diff --git a/.config/ci.yml b/.config/ci.yml index 5fcf78b737..8543205b17 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -374,9 +374,6 @@ attachLdSignatureForRelays: true # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false -# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. -# # default: false in production, true otherwise. -# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index eae7e14308..f705d06d45 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -325,9 +325,6 @@ allowedPrivateNetworks: # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false -# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. -# # default: false in production, true otherwise. -# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/.config/docker_example.yml b/.config/docker_example.yml index de43d3e43c..5905e3deed 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -442,9 +442,6 @@ attachLdSignatureForRelays: true # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false -# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. -# # default: false in production, true otherwise. -# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/.config/example.yml b/.config/example.yml index 745004305d..cffc333d14 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -448,9 +448,6 @@ attachLdSignatureForRelays: true # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false -# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable. -# # default: false in production, true otherwise. -# #verbose: false # Settings for the activity logger, which records inbound activities to the database. # Disabled by default due to the large volume of data it saves. diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c97d9ba04a..31be935c47 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -128,13 +128,10 @@ backend_tests: - name: redis pull_policy: if-not-present script: - - >- - pnpm run build \ - --filter=backend \ - --filter=megalodon \ - --filter=misskey-js - - pnpm run migrate - - pnpm run test --filter=backend + - pnpm run --filter backend build:pre + - pnpm run --filter backend build + - pnpm run --filter backend migrate + - pnpm run --filter backend test # Same as common, but MRs are only run if they modify the backend. rules: - if: $CI_PIPELINE_SOURCE == 'push' && ($CI_COMMIT_BRANCH == 'develop' || $CI_COMMIT_BRANCH == 'stable') @@ -152,16 +149,9 @@ backend_tests: frontend_tests: <<: *test_common script: - - >- - pnpm run build \ - --filter=frontend \ - --filter=frontend-embed \ - --filter=frontend-shared \ - --filter=misskey-js \ - --filter=misskey-bubble-game \ - --filter=misskey-reversi \ - --filter=sw - - pnpm run test --filter=frontend --filter=misskey-js + - pnpm run --filter frontend build:pre + - pnpm run --filter frontend build + - pnpm run --filter frontend test # Same as common, but MRs are only run if they modify the frontend. rules: - if: $CI_PIPELINE_SOURCE == 'push' && ($CI_COMMIT_BRANCH == 'develop' || $CI_COMMIT_BRANCH == 'stable') @@ -182,6 +172,18 @@ frontend_tests: - 'cypress/**/*' - 'assets/**/*' +misskey-js_tests: + <<: *test_common + script: + - pnpm run --filter misskey-js build + - pnpm run --filter misskey-js test + +megalodon_tests: + <<: *test_common + script: + - pnpm run --filter megalodon build + - pnpm run --filter megalodon test + get_image_tag: <<: *deploy_common image: diff --git a/CONTRIBUTING.Sharkey.md b/CONTRIBUTING.Sharkey.md index b4e6a90270..b20a3fdf9b 100644 --- a/CONTRIBUTING.Sharkey.md +++ b/CONTRIBUTING.Sharkey.md @@ -326,11 +326,14 @@ git checkout -m merge/$(date +%Y-%m-%d) # Create/switch to a merge branch for git merge --no-ff misskey/develop # Merge from Misskey's develop branch, forcing a merge commit. ``` -Fix conflicts and *commit!* Conflicts in `pnpm-lock.yaml` can usually -be fixed by running `pnpm install` - it detects conflict markers and -seems to do a decent job. +Fix conflicts and *commit!* +- Conflicts in `pnpm-lock.yaml` can be fixed by rejecting changes and running `pnpm install`. +- Conflicts in `packages/misskey-js/etc` and `packages/misskey-js/src/autogen` can be fixed by rejecting changes and running `pnpm run build-misskey-js-with-types`. +- Conflicts in `locales/index.d.ts` can be fixed by rejecting changes and running `pnpm run build-assets`. +- Conflicts in any `package.json` file can be fixed by merging only added/removed dependencies, then running `pnpm run sync-dependency-versions`. Other changes (not dependencies) will need to be merged manually. +- Conflicts involving `this.timeService.now` or `this.timeService.date` can be resolved by accepting remote changes. ESLint will highlight all the missing references in a later step. -*After that commit,* do all the extra work, on the same branch: +*After that commit*, do all the extra work on the same branch: - Copy all changes (commit after each step): - in `packages/backend/src/core/activitypub/models/ApNoteService.ts`, from `createNote` to `updateNote` @@ -358,14 +361,44 @@ seems to do a decent job. - from `.config/example.yml` to `.config/ci.yml` and `chart/files/default.yml` - in `packages/backend/src/core/MfmService.ts`, from `toHtml` to `toMastoApiHtml` - from `verifyLink` in `packages/backend/src/core/activitypub/models/ApPersonService.ts` to `verifyFieldLinks` in `packages/backend/src/misc/verify-field-link.ts` (if sensible) - -- If there have been any changes to the federated user data (the - `renderPerson` function in - `packages/backend/src/core/activitypub/ApRendererService.ts`), make - sure that the set of fields in `userNeedsPublishing` and - `profileNeedsPublishing` in - `packages/backend/src/server/api/endpoints/i/update.ts` are still - correct. +- Check for changes that may require additional work: + - If there have been any changes to the federated user data (the + `renderPerson` function in + `packages/backend/src/core/activitypub/ApRendererService.ts`), make + sure that the set of fields in `userNeedsPublishing` and + `profileNeedsPublishing` in + `packages/backend/src/server/api/endpoints/i/update.ts` are still + correct. + - Check for any new instances of any memory cache class. + (`MemoryKVCache`, `MemorySingleCache`, `RedisKVCache`, `RedisSingleCache`, and `QuantumKVCache` are the current ones.) + These can usually be kept as-is, but all instances must be managed by `CacheManagementService`. + The conversion is easy: + 1. Make sure that `CacheManagementService` is available. + In most cases, it can be injected through DI. + (it's in the `GlobalModule` which should be available everywhere.) + 2. Find where the cache is constructed. + If it's a field initializer, then move it to the constructor (splitting declaration and initialization.) + 3. Replace the `new Whatever()` statement with a call to `cacheManagementService.createWhatever()`. + Arguments can be kept as-is, but remove any references to `Redis`, `InternalEventService`, or `TimeService`. + (these are provided by `CacheManagementService` directly.) + 4. Remove any calls to `dispose()` the cache. + Disposal is managed by `CacheManagementService`, so attempting to call any `dispose` or `onApplicationShutdown` method will produce a type error. + - Check for any new calls to native time functions: + - `Date.now()` - replace with `this.timeService.now`. + Inject `TimeService` via DI if it's not already available. + - `new Date()` - if there's a value passed in, then leave it. + But the no-args constructor should be replaced with `this.timeService.date`. + Inject `TimeService` via DI if it's not already available. + - `setTimeout` - migrate to `this.timeService.startTimer` or `this.timeService.startPromiseTimer`. + The parameters should be the same, but the return type is different. + You may need to replace some `NodeJS.Timeout` types with `this.timerHandle`. + Inject `TimeService` via DI if it's not already available. + - `setInterval` - migrate to `this.timeService.startTimer`. + Migration is mostly the same as `setTimeout`, but with one major difference: + You must add `{ repeated: true }` as the final option parameter. + If this is omitted, the code will compile but the interval will only fire once! + - Check for any new Chart subclasses, and make sure to inject `TimeService` and implement `getCurrentDate`. + - Check for any new Channel subclasses and add all missing DI parameters. - Check the changes against our `develop` branch (`git diff develop`) and against Misskey's `develop` branch (`git diff misskey/develop`). @@ -465,3 +498,6 @@ following apply: together. Using `MemorySingleCache` or `RedisSingleCache` could provide a cleaner implementation without resorting to hacks like a fixed key. + +- It's necessary to use `null` as a data value. + `QuantumKVCache` does not allow null values, and thus another option should be chosen. diff --git a/cypress/tsconfig.json b/cypress/tsconfig.json index 4435a4fda8..bff0483305 100644 --- a/cypress/tsconfig.json +++ b/cypress/tsconfig.json @@ -1,9 +1,8 @@ { + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../packages/shared/tsconfig.web.jsonc", "compilerOptions": { - "lib": ["dom", "es5"], - "target": "es5", "types": ["cypress", "node"], - "incremental": true }, "include": ["./**/*.ts"] } diff --git a/locales/package.json b/locales/package.json index bedb411a91..ac5d3b4422 100644 --- a/locales/package.json +++ b/locales/package.json @@ -1,3 +1,5 @@ { - "type": "module" + "type": "module", + "main": "index.js", + "types": "index.d.ts" } diff --git a/package.json b/package.json index 7b93e9416e..05ef62da35 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "type": "git", "url": "https://activitypub.software/TransFem-org/Sharkey.git" }, - "packageManager": "pnpm@9.6.0", + "packageManager": "pnpm@10.16.0", "workspaces": [ "packages/frontend-shared", "packages/frontend", @@ -36,7 +36,7 @@ "lint": "pnpm -r lint", "lint-all": "pnpm -r --no-bail lint", "eslint": "pnpm -r eslint", - "eslint-all": "pnpm -r --no-bail eslint", + "eslint-all": "pnpm -r --no-bail eslint-all && pnpm -r --no-bail eslint", "cy:open": "pnpm cypress open --browser --e2e --config-file=cypress.config.ts", "cy:run": "pnpm cypress run", "e2e": "pnpm start-server-and-test start:test http://localhost:61812 cy:run", @@ -47,38 +47,40 @@ "test-and-coverage": "pnpm -r test-and-coverage", "clean": "node ./scripts/clean.js", "clean-all": "node ./scripts/clean-all.js", - "cleanall": "pnpm clean-all" + "cleanall": "pnpm clean-all", + "sync-dependency-versions": "node scripts/sync-deps.mjs" }, "resolutions": { "chokidar": "4.0.3", - "lodash": "4.17.21" + "lodash": "4.17.21", + "axios": "1.12.2" }, "dependencies": { "js-yaml": "4.1.0" }, "optionalDependencies": { - "cypress": "14.3.2" + "cypress": "15.3.0" }, "devDependencies": { "@misskey-dev/eslint-plugin": "2.1.0", - "@types/node": "22.15.2", - "@typescript-eslint/eslint-plugin": "8.31.0", - "@typescript-eslint/parser": "8.31.0", - "cross-env": "7.0.3", - "cssnano": "7.0.6", - "esbuild": "0.25.3", - "eslint": "9.25.1", - "execa": "9.5.2", + "@types/node": "22.18.1", + "@typescript-eslint/eslint-plugin": "8.44.1", + "@typescript-eslint/parser": "8.44.1", + "cross-env": "10.0.0", + "cssnano": "7.1.1", + "esbuild": "0.25.10", + "eslint": "9.36.0", + "execa": "9.6.0", "fast-glob": "3.3.3", - "glob": "11.0.2", - "globals": "16.1.0", + "glob": "11.0.3", + "globals": "16.4.0", + "ignore-walk": "8.0.0", "ncp": "2.0.0", - "pnpm": "9.6.0", - "ignore-walk": "7.0.0", - "postcss": "8.5.3", - "start-server-and-test": "2.0.11", - "tar": "7.4.3", - "terser": "5.39.0", - "typescript": "5.8.3" + "pnpm": "10.17.1", + "postcss": "8.5.6", + "start-server-and-test": "2.1.2", + "tar": "7.5.1", + "terser": "5.44.0", + "typescript": "5.9.2" } } diff --git a/packages/backend/.swcrc b/packages/backend/.swcrc index f4bf7a4d2a..758045025f 100644 --- a/packages/backend/.swcrc +++ b/packages/backend/.swcrc @@ -17,7 +17,7 @@ "paths": { "@/*": ["*"] }, - "target": "es2022" + "target": "ESNext" }, "minify": false, "sourceMaps": "inline" diff --git a/packages/backend/assets/embed.js b/packages/backend/assets/embed.js index 24fccc1b6c..38b6133847 100644 --- a/packages/backend/assets/embed.js +++ b/packages/backend/assets/embed.js @@ -7,7 +7,7 @@ /** @type {NodeListOf} */ const els = document.querySelectorAll('iframe[data-misskey-embed-id]'); - window.addEventListener('message', function (event) { + window.addEventListener('message', (event) => { els.forEach((el) => { if (event.source !== el.contentWindow) { return; diff --git a/packages/backend/eslint.config.js b/packages/backend/eslint.config.js index 2531d9bae4..56a49fb2fc 100644 --- a/packages/backend/eslint.config.js +++ b/packages/backend/eslint.config.js @@ -5,7 +5,20 @@ import sharedConfig from '../shared/eslint.config.js'; export default [ ...sharedConfig, { - ignores: ['**/node_modules', 'built', '@types/**/*', 'migration'], + ignores: [ + '**/built/', + 'migration/', + '**/node_modules/', + 'test/', + 'test-federation/', + 'test-server/', + '**/temp/', + '**/@types/', + '**/coverage/', + 'ormconfig.js', + 'scripts/check_connect.js', + 'scripts/generate_api_json.js', + ], }, { languageOptions: { @@ -15,11 +28,15 @@ export default [ }, }, { - files: ['**/*.ts', '**/*.tsx'], + files: ['src/**/*.ts', 'src/**/*.tsx'], + ignores: [ + '*.*', + 'src/server/web/**/*.d.ts', + ], languageOptions: { parserOptions: { parser: tsParser, - project: ['./tsconfig.json', './test/tsconfig.json', './test-federation/tsconfig.json'], + project: ['./tsconfig.backend.json'], sourceType: 'module', tsconfigRootDir: import.meta.dirname, }, @@ -42,17 +59,89 @@ export default [ position: 'after', }], }], - 'no-restricted-globals': ['error', { - name: '__dirname', - message: 'Not in ESModule. Use `import.meta.url` instead.', - }, { - name: '__filename', - message: 'Not in ESModule. Use `import.meta.url` instead.', - }], + 'no-restricted-globals': [ + 'error', + { + globals: [ + { + name: '__dirname', + message: 'Not in ESModule. Use `import.meta.url` instead.', + }, + { + name: '__filename', + message: 'Not in ESModule. Use `import.meta.url` instead.', + }, + { + name: 'setTimeout', + message: 'Use TimeService.startTimer instead.', + }, + { + name: 'setInterval', + message: 'Use TimeService.startTimer instead.', + }, + { + name: 'console', + message: 'Use a Logger instance instead.', + }, + ], + checkGlobalObject: true, + }, + ], + 'no-restricted-properties': [ + 'error', + { + object: 'Date', + property: 'now', + message: 'Use TimeService.now instead.', + }, + ], + 'no-restricted-syntax': [ + 'error', + { + 'selector': 'NewExpression[callee.name=\'Date\'][arguments.length=0]', + 'message': 'new Date() is restricted. Use TimeService.date instead.', + }, + { + 'selector': 'NewExpression[callee.name=\'MemoryKVCache\']', + 'message': 'Cache constructor will produce an unmanaged instance. Use CacheManagementService.createMemoryKVCache() instead.', + }, + { + 'selector': 'NewExpression[callee.name=\'MemorySingleCache\']', + 'message': 'Cache constructor will produce an unmanaged instance. Use CacheManagementService.createMemorySingleCache() instead.', + }, + { + 'selector': 'NewExpression[callee.name=\'RedisKVCache\']', + 'message': 'Cache constructor will produce an unmanaged instance. Use CacheManagementService.createRedisKVCache() instead.', + }, + { + 'selector': 'NewExpression[callee.name=\'RedisSingleCache\']', + 'message': 'Cache constructor will produce an unmanaged instance. Use CacheManagementService.createRedisSingleCache() instead.', + }, + { + 'selector': 'NewExpression[callee.name=\'QuantumKVCache\']', + 'message': 'Cache constructor will produce an unmanaged instance. Use CacheManagementService.createQuantumKVCache() instead.', + }, + { + 'selector': 'CallExpression[callee.property.name=\'delete\'][arguments.length=1] > ObjectExpression[properties.length=0]', + 'message': 'repository.delete({}) will produce a runtime error. Use repository.deleteAll() instead.', + }, + { + 'selector': 'CallExpression[callee.property.name=\'update\'][arguments.length>=1] > ObjectExpression[properties.length=0]', + 'message': 'repository.update({}, {...}) will produce a runtime error. Use repository.updateAll({...}) instead.', + }, + ], }, }, { - files: ['src/server/web/**/*.js', 'src/server/web/**/*.ts'], + files: [ + './assets/**/*.js', + './assets/**/*.mjs', + './assets/**/*.cjs', + './src/server/web/**/*.js', + './src/server/web/**/*.mjs', + './src/server/web/**/*.cjs', + './src/server/web/**/*.d.ts', + ], languageOptions: { globals: { ...globals.browser, @@ -60,16 +149,41 @@ export default [ CLIENT_ENTRY: true, LANGS_VERSION: true, }, + parserOptions: { + parser: tsParser, + project: ['./tsconfig.frontend.json'], + sourceType: 'module', + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + 'no-restricted-globals': 'off', + 'no-restricted-properties': 'off', + 'no-restricted-syntax': 'off', }, }, { + files: [ + 'eslint.*', + 'jest.*', + 'scripts/dev.mjs', + 'scripts/watch.mjs', + ], ignores: [ - "**/lib/", - "**/temp/", - "**/built/", - "**/coverage/", - "**/node_modules/", - "**/migration/", - ] + 'ormconfig.js', + 'scripts/check_connect.js', + 'scripts/generate_api_json.js', + ], + languageOptions: { + parserOptions: { + parser: tsParser, + project: ['./tsconfig.scripts.json'], + sourceType: 'module', + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + 'import/no-default-export': 'off', + }, }, ]; diff --git a/packages/backend/jest.config.cjs b/packages/backend/jest.config.common.ts similarity index 97% rename from packages/backend/jest.config.cjs rename to packages/backend/jest.config.common.ts index 5a4aa4e15a..18954c9dd0 100644 --- a/packages/backend/jest.config.cjs +++ b/packages/backend/jest.config.common.ts @@ -3,7 +3,7 @@ * https://jestjs.io/docs/en/configuration.html */ -module.exports = { +export default { // All imported modules in your tests should be mocked automatically // automock: false, @@ -91,7 +91,7 @@ module.exports = { // See https://github.com/swc-project/jest/issues/64#issuecomment-1029753225 // TODO: Use `--allowImportingTsExtensions` on TypeScript 5.0 so that we can // directly import `.ts` files without this hack. - '^((?:\\.{1,2}|[A-Z:])*/.*)\\.js$': '$1', + '^(\\.{1,2}[\\/\\\\].*)\\.js$': '$1', }, // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader @@ -129,7 +129,9 @@ module.exports = { // A list of paths to directories that Jest should use to search for files in roots: [ - "" + '/src', + '/test', + '/test-federation', ], // Allows you to use a custom runner instead of Jest's default test runner @@ -142,7 +144,7 @@ module.exports = { // setupFilesAfterEnv: [], // The number of seconds after which a test is considered as slow and reported as such in the results. - // slowTestThreshold: 5, + slowTestThreshold: 5, // A list of paths to snapshot serializer modules Jest should use for snapshot testing // snapshotSerializers: [], @@ -157,10 +159,7 @@ module.exports = { // testLocationInResults: false, // The glob patterns Jest uses to detect test files - testMatch: [ - "/test/unit/**/*.ts", - "/src/**/*.test.ts", - ], + testMatch: [], // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped // testPathIgnorePatterns: [ diff --git a/packages/backend/jest.config.e2e.cjs b/packages/backend/jest.config.e2e.ts similarity index 67% rename from packages/backend/jest.config.e2e.cjs rename to packages/backend/jest.config.e2e.ts index 4502da47df..3a77ad02b6 100644 --- a/packages/backend/jest.config.e2e.cjs +++ b/packages/backend/jest.config.e2e.ts @@ -3,12 +3,12 @@ * https://jestjs.io/docs/en/configuration.html */ -const base = require('./jest.config.cjs') +import base from './jest.config.common.ts'; -module.exports = { +export default { ...base, globalSetup: "/built-test/entry.js", - setupFilesAfterEnv: ["/test/jest.setup.ts"], + setupFilesAfterEnv: ["/test/jest.setup.e2e.mjs"], testMatch: [ "/test/e2e/**/*.ts", ], diff --git a/packages/backend/jest.config.fed.cjs b/packages/backend/jest.config.fed.ts similarity index 84% rename from packages/backend/jest.config.fed.cjs rename to packages/backend/jest.config.fed.ts index fae187bc23..a8dd950bff 100644 --- a/packages/backend/jest.config.fed.cjs +++ b/packages/backend/jest.config.fed.ts @@ -3,7 +3,7 @@ * https://jestjs.io/docs/en/configuration.html */ -const base = require('./jest.config.cjs'); +import base from './jest.config.common.ts'; module.exports = { ...base, diff --git a/packages/backend/jest.config.unit.cjs b/packages/backend/jest.config.unit.ts similarity index 68% rename from packages/backend/jest.config.unit.cjs rename to packages/backend/jest.config.unit.ts index aa5992936b..42dd6bfe96 100644 --- a/packages/backend/jest.config.unit.cjs +++ b/packages/backend/jest.config.unit.ts @@ -3,10 +3,11 @@ * https://jestjs.io/docs/en/configuration.html */ -const base = require('./jest.config.cjs') +import base from './jest.config.common.ts'; -module.exports = { +export default { ...base, + globalSetup: '/test/jest.setup.unit.mjs', testMatch: [ "/test/unit/**/*.ts", "/src/**/*.test.ts", diff --git a/packages/backend/jest.js b/packages/backend/jest.js index 0e761d8c92..0f28dc2f80 100644 --- a/packages/backend/jest.js +++ b/packages/backend/jest.js @@ -24,7 +24,11 @@ child.on('error', (err) => { }); child.on('exit', (code, signal) => { if (code === null) { - process.exit(128 + signal); + if (signal != null) { + process.exit(128 + signal); + } else { + process.exit(128); + } } else { process.exit(code); } diff --git a/packages/backend/jsconfig.json b/packages/backend/jsconfig.json deleted file mode 100644 index 1230aadd12..0000000000 --- a/packages/backend/jsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "es6", - "module": "commonjs", - "allowSyntheticDefaultImports": true - }, - "exclude": [ - "node_modules", - "jspm_packages", - "tmp", - "temp" - ] -} diff --git a/packages/backend/migration/1740162088574-add_unsignedFetch.js b/packages/backend/migration/1740162088574-add_unsignedFetch.js index 0744e68dfa..a30f5ae966 100644 --- a/packages/backend/migration/1740162088574-add_unsignedFetch.js +++ b/packages/backend/migration/1740162088574-add_unsignedFetch.js @@ -1,4 +1,7 @@ import { loadConfig } from '../built/config.js'; +import { LoggerService } from '../built/core/LoggerService.js'; +import { NativeTimeService } from '../built/global/TimeService.js'; +import { EnvService } from '../built/global/EnvService.js'; export class AddUnsignedFetch1740162088574 { name = 'AddUnsignedFetch1740162088574' @@ -16,7 +19,8 @@ export class AddUnsignedFetch1740162088574 { await queryRunner.query(`UPDATE "user" SET "allowUnsignedFetch" = 'always' WHERE "username" LIKE '%.%' AND "host" IS null`); // Special one-time migration: convert legacy config "" to meta setting "" - const config = await loadConfig(); + const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService()); + const config = await loadConfig(loggerService); if (config.checkActivityPubGetSignature) { // noinspection SqlWithoutWhere await queryRunner.query(`UPDATE "meta" SET "allowUnsignedFetch" = 'never'`); diff --git a/packages/backend/ormconfig.js b/packages/backend/ormconfig.js index d59e647d89..14a66cdf57 100644 --- a/packages/backend/ormconfig.js +++ b/packages/backend/ormconfig.js @@ -2,8 +2,12 @@ import { DataSource } from 'typeorm'; import { loadConfig } from './built/config.js'; import { entities } from './built/postgres.js'; import { isConcurrentIndexMigrationEnabled } from "./migration/js/migration-config.js"; +import { LoggerService } from './built/core/LoggerService.js'; +import { NativeTimeService } from './built/global/TimeService.js'; +import { EnvService } from './built/global/EnvService.js'; -const config = loadConfig(); +const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService()); +const config = loadConfig(loggerService); export default new DataSource({ type: 'postgres', diff --git a/packages/backend/package.json b/packages/backend/package.json index 88244db346..bfa32c8612 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -16,41 +16,53 @@ "revert": "pnpm typeorm migration:revert -d ormconfig.js", "check:connect": "node ./scripts/check_connect.js", "build": "swc src -d built -D --strip-leading-paths", + "build:pre": "pnpm run -w build-pre && pnpm run --filter megalodon build && pnpm run --filter misskey-js build && pnpm run --filter misskey-reversi build && pnpm run --filter misskey-bubble-game build && pnpm run -w build-assets", "build:test": "swc test-server -d built-test -D --config-file test-server/.swcrc --strip-leading-paths", - "watch:swc": "swc src -d built -D -w --strip-leading-paths", - "build:tsc": "tsc -p tsconfig.json && tsc-alias -p tsconfig.json", - "watch": "node ./scripts/watch.mjs", + "build:tsc": "tsc -p tsconfig.backend.json && tsc-alias -p tsconfig.backend.json", + "watch": "pnpm run build:pre && node ./scripts/watch.mjs", "restart": "pnpm build && pnpm start", - "dev": "node ./scripts/dev.mjs", - "typecheck": "pnpm --filter megalodon build && tsc --noEmit && tsc -p test --noEmit && tsc -p test-federation --noEmit", - "eslint": "eslint --quiet \"{src,test-federation,test,js,@types}/**/*.{js,jsx,ts,tsx,vue}\" --cache", + "dev": "pnpm run build:pre && node ./scripts/dev.mjs", + "typecheck-all": "pnpm run --no-bail build:pre && pnpm run --no-bail typecheck:scripts && pnpm run --no-bail typecheck:backend && pnpm run --no-bail typecheck:frontend && pnpm run --no-bail typecheck:test && pnpm run --no-bail typecheck:test-federation && pnpm run --no-bail typecheck:test-server", + "typecheck": "pnpm run build:pre && pnpm run typecheck:scripts && pnpm run typecheck:backend && pnpm run typecheck:frontend && pnpm run typecheck:test && pnpm run typecheck:test-federation && pnpm run typecheck:test-server", + "typecheck:scripts": "tsc -p tsconfig.scripts.json --noEmit", + "typecheck:backend": "tsc -p tsconfig.backend.json --noEmit", + "typecheck:frontend": "tsc -p tsconfig.frontend.json --noEmit", + "typecheck:test": "tsc -p test/tsconfig.json --noEmit", + "typecheck:test-federation": "tsc -p test-federation/tsconfig.json --noEmit", + "typecheck:test-server": "tsc -p test-server/tsconfig.json --noEmit", + "eslint-all": "pnpm run --no-bail eslint:backend && pnpm run --no-bail eslint:test && pnpm run --no-bail eslint:test-federation && pnpm run --no-bail eslint:test-server", + "eslint": "pnpm run eslint:backend && pnpm run eslint:test && pnpm run eslint:test-federation && pnpm run eslint:test-server", + "eslint:backend": "eslint --quiet --cache -c eslint.config.js .", + "eslint:test": "eslint --quiet --cache -c test/eslint.config.js ./test", + "eslint:test-federation": "eslint --quiet --cache -c test-federation/eslint.config.js ./test-federation", + "eslint:test-server": "eslint --quiet --cache -c test-server/eslint.config.js ./test-server", "lint": "pnpm typecheck && pnpm eslint", - "jest": "cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.unit.cjs", - "jest:e2e": "cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.e2e.cjs", - "jest:fed": "node ./jest.js --forceExit --config jest.config.fed.cjs", - "jest-and-coverage": "cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.unit.cjs", - "jest-and-coverage:e2e": "cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.e2e.cjs", + "jest": "cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.unit.ts", + "jest:e2e": "cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.e2e.ts", + "jest:fed": "node ./jest.js --forceExit --config jest.config.fed.ts", + "jest-and-coverage": "cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.unit.ts", + "jest-and-coverage:e2e": "cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.e2e.ts", "jest-clear": "cross-env NODE_ENV=test node ./jest.js --clearCache", "test": "pnpm jest", - "test:e2e": "pnpm build && pnpm build:test && pnpm jest:e2e", - "test:fed": "pnpm jest:fed", - "test-and-coverage": "pnpm jest-and-coverage", - "test-and-coverage:e2e": "pnpm build && pnpm build:test && pnpm jest-and-coverage:e2e", + "test:e2e": "pnpm run build && pnpm run build:test && pnpm run jest:e2e", + "test:fed": "pnpm run build && pnpm run build:test && pnpm run jest:fed", + "test-and-coverage": "pnpm run build && pnpm run build:test && pnpm run jest-and-coverage", + "test-and-coverage:e2e": "pnpm run build && pnpm run build:test && pnpm run jest-and-coverage:e2e", "generate-api-json": "node ./scripts/generate_api_json.js" }, "optionalDependencies": { "@swc/core-android-arm64": "1.3.11", - "@swc/core-darwin-arm64": "1.11.24", - "@swc/core-darwin-x64": "1.11.24", + "@swc/core-darwin-arm64": "1.13.5", + "@swc/core-darwin-x64": "1.13.5", "@swc/core-freebsd-x64": "1.3.11", - "@swc/core-linux-arm-gnueabihf": "1.11.24", - "@swc/core-linux-arm64-gnu": "1.11.24", - "@swc/core-linux-arm64-musl": "1.11.24", - "@swc/core-linux-x64-gnu": "1.11.24", - "@swc/core-linux-x64-musl": "1.11.24", - "@swc/core-win32-arm64-msvc": "1.11.24", - "@swc/core-win32-ia32-msvc": "1.11.24", - "@swc/core-win32-x64-msvc": "1.11.24", + "@swc/core-linux-arm-gnueabihf": "1.13.5", + "@swc/core-linux-arm64-gnu": "1.13.5", + "@swc/core-linux-arm64-musl": "1.13.5", + "@swc/core-linux-x64-gnu": "1.13.5", + "@swc/core-linux-x64-musl": "1.13.5", + "@swc/core-win32-arm64-msvc": "1.13.5", + "@swc/core-win32-ia32-msvc": "1.13.5", + "@swc/core-win32-x64-msvc": "1.13.5", "bufferutil": "4.0.9", "slacc-android-arm-eabi": "0.0.10", "slacc-android-arm64": "0.0.10", @@ -68,86 +80,84 @@ "utf-8-validate": "6.0.5" }, "dependencies": { - "@aws-sdk/client-s3": "3.797.0", - "@aws-sdk/lib-storage": "3.797.0", - "@discordapp/twemoji": "15.1.0", + "@aws-sdk/client-s3": "3.896.0", + "@aws-sdk/lib-storage": "3.896.0", + "@discordapp/twemoji": "16.0.1", "@fastify/accepts": "5.0.2", "@fastify/cookie": "11.0.2", - "@fastify/cors": "10.1.0", + "@fastify/cors": "11.1.0", "@fastify/express": "4.0.2", - "@fastify/http-proxy": "10.0.2", - "@fastify/multipart": "9.0.3", - "@fastify/static": "8.1.1", - "@fastify/view": "10.0.2", + "@fastify/http-proxy": "11.3.0", + "@fastify/multipart": "9.2.1", + "@fastify/static": "8.2.0", + "@fastify/view": "11.1.1", "@misskey-dev/sharp-read-bmp": "1.3.0", - "@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.2", - "@nestjs/common": "11.1.0", - "@nestjs/core": "11.1.0", - "@nestjs/testing": "11.1.0", + "@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.3", + "@nestjs/common": "11.1.6", + "@nestjs/core": "11.1.6", + "@nestjs/testing": "11.1.6", "@peertube/http-signature": "1.7.0", - "@sentry/node": "8.55.0", - "@sentry/profiling-node": "8.55.0", + "@sentry/node": "10.15.0", + "@sentry/profiling-node": "10.15.0", "@simplewebauthn/server": "12.0.0", - "@sinonjs/fake-timers": "11.3.1", - "@smithy/node-http-handler": "2.5.0", - "mfm-js": "npm:@transfem-org/sfm-js@0.24.8", - "@twemoji/parser": "15.1.1", + "@smithy/node-http-handler": "4.2.1", "accepts": "1.3.8", "ajv": "8.17.1", "archiver": "7.0.1", - "argon2": "0.43.0", - "axios": "1.7.4", - "bcryptjs": "2.4.3", + "argon2": "0.44.0", + "axios": "1.12.2", + "bcryptjs": "3.0.2", "blurhash": "2.0.5", - "bullmq": "5.51.1", + "bullmq": "5.58.7", "cacheable-lookup": "7.0.0", - "canvas": "3.1.0", - "cbor": "9.0.2", - "chalk": "5.4.1", - "chalk-template": "1.1.0", - "cheerio": "1.0.0", - "cli-highlight": "npm:@transfem-org/cli-highlight@2.1.12", - "color-convert": "2.0.1", + "canvas": "3.2.0", + "cbor": "10.0.11", + "chalk": "5.6.2", + "chalk-template": "1.1.2", + "cheerio": "1.1.2", + "cli-highlight": "npm:@transfem-org/cli-highlight@2.1.13", + "color-convert": "3.1.2", "content-disposition": "0.5.4", - "date-fns": "2.30.0", + "date-fns": "4.1.0", "deep-email-validator": "0.1.21", "dom-serializer": "2.0.0", "domhandler": "5.0.3", "domutils": "3.2.2", - "fastify": "5.3.2", + "fastify": "5.6.1", "fastify-raw-body": "5.0.0", - "feed": "4.2.2", - "file-type": "19.6.0", + "feed": "5.1.0", + "file-type": "21.0.0", "fluent-ffmpeg": "2.1.3", - "form-data": "4.0.2", - "glob": "11.0.0", - "got": "14.4.7", + "form-data": "4.0.4", + "glob": "11.0.3", + "got": "14.4.9", "hpagent": "1.2.0", "htmlescape": "1.1.1", - "htmlparser2": "9.1.0", - "ioredis": "5.6.1", + "htmlparser2": "10.0.0", + "ioredis": "5.8.0", "ip-cidr": "4.0.2", "ipaddr.js": "2.2.0", - "is-svg": "5.1.0", + "is-svg": "6.1.0", "js-yaml": "4.1.0", "json5": "2.2.3", "jsonld": "8.3.3", "juice": "11.0.1", "megalodon": "workspace:*", - "meilisearch": "0.50.0", - "mime-types": "2.1.35", + "meilisearch": "0.53.0", + "mfm-js": "npm:@transfem-org/sfm-js@0.26.1", + "mime-types": "3.0.1", "misskey-js": "workspace:*", "misskey-reversi": "workspace:*", "moment": "2.30.1", - "ms": "3.0.0-canary.1", - "nanoid": "5.1.5", + "ms": "3.0.0-canary.202508261828", + "nanoid": "5.1.6", "nested-property": "4.0.0", "node-fetch": "3.3.2", - "nodemailer": "6.10.1", + "nodemailer": "7.0.6", "os-utils": "0.0.14", - "otpauth": "9.4.0", - "pg": "8.15.6", - "pkce-challenge": "4.1.0", + "otpauth": "9.4.1", + "pg": "8.16.3", + "pkce-challenge": "5.0.0", "probe-image-size": "7.2.3", "promise-limit": "2.7.0", "proxy-addr": "2.0.7", @@ -155,58 +165,57 @@ "pug": "3.0.3", "qrcode": "1.5.4", "random-seed": "0.3.0", - "re2": "1.21.4", + "re2": "1.22.1", "redis-info": "3.1.0", - "redis-lock": "0.1.4", + "redis-lock": "1.0.0", "reflect-metadata": "0.2.2", "rename": "1.0.4", - "sanitize-html": "2.16.0", - "secure-json-parse": "3.0.2", - "sharp": "0.34.1", - "semver": "7.7.1", + "sanitize-html": "2.17.0", + "secure-json-parse": "4.0.0", + "semver": "7.7.2", + "sharp": "0.34.4", "slacc": "0.0.10", "strict-event-emitter-types": "2.0.0", - "systeminformation": "5.25.11", + "systeminformation": "5.27.10", "tinycolor2": "1.6.0", - "tmp": "0.2.3", - "tsc-alias": "1.8.15", + "tmp": "0.2.5", + "tsc-alias": "1.8.16", "tsconfig-paths": "4.2.0", - "typeorm": "0.3.22", - "typescript": "5.8.3", - "ulid": "2.4.0", - "uuid": "11.1.0", + "typeorm": "0.3.27", + "ulid": "3.0.1", + "uuid": "13.0.0", "vary": "1.1.2", "web-push": "3.6.7", - "ws": "8.18.1", + "ws": "8.18.3", "xev": "3.0.2" }, "devDependencies": { - "@jest/globals": "29.7.0", - "@nestjs/platform-express": "11.1.0", - "@sentry/vue": "9.14.0", + "@jest/globals": "30.2.0", + "@nestjs/platform-express": "11.1.6", + "@sentry/vue": "10.15.0", "@simplewebauthn/types": "12.0.0", - "@swc/cli": "0.7.3", - "@swc/core": "1.11.24", - "@swc/jest": "0.2.38", + "@swc/cli": "0.7.8", + "@swc/core": "1.13.5", + "@swc/jest": "0.2.39", "@types/accepts": "1.3.7", "@types/archiver": "6.0.3", - "@types/bcryptjs": "2.4.6", + "@types/bcryptjs": "3.0.0", "@types/color-convert": "2.0.4", - "@types/content-disposition": "0.5.8", + "@types/content-disposition": "0.5.9", "@types/fluent-ffmpeg": "2.1.27", "@types/htmlescape": "1.1.3", - "@types/jest": "29.5.14", + "@types/jest": "30.0.0", "@types/js-yaml": "4.0.9", "@types/jsonld": "1.5.15", "@types/jsrsasign": "10.5.15", - "@types/mime-types": "2.1.4", - "@types/ms": "0.7.34", - "@types/node": "22.15.2", - "@types/nodemailer": "6.4.17", + "@types/mime-types": "3.0.1", + "@types/ms": "2.1.0", + "@types/node": "22.18.1", + "@types/nodemailer": "7.0.1", "@types/oauth": "0.9.6", "@types/oauth2orize": "1.11.5", "@types/oauth2orize-pkce": "0.1.2", - "@types/pg": "8.11.14", + "@types/pg": "8.15.5", "@types/proxy-addr": "2.0.3", "@types/psl": "1.1.3", "@types/pug": "2.0.10", @@ -214,28 +223,32 @@ "@types/random-seed": "0.3.5", "@types/redis-info": "3.0.3", "@types/rename": "1.0.7", - "@types/sanitize-html": "2.15.0", - "@types/semver": "7.7.0", + "@types/sanitize-html": "2.16.0", + "@types/semver": "7.7.1", "@types/simple-oauth2": "5.0.7", - "@types/sinonjs__fake-timers": "8.1.5", "@types/supertest": "6.0.3", "@types/tinycolor2": "1.4.6", "@types/tmp": "0.2.6", "@types/vary": "1.1.3", "@types/web-push": "3.6.4", "@types/ws": "8.18.1", - "@typescript-eslint/eslint-plugin": "8.31.0", - "@typescript-eslint/parser": "8.31.0", + "@typescript-eslint/eslint-plugin": "8.44.1", + "@typescript-eslint/parser": "8.44.1", "aws-sdk-client-mock": "4.1.0", - "cross-env": "7.0.3", - "eslint-plugin-import": "2.31.0", - "execa": "9.5.2", + "cross-env": "10.0.0", + "eslint": "9.36.0", + "eslint-plugin-import": "2.32.0", + "execa": "9.6.0", "fkill": "9.0.0", - "jest": "29.7.0", - "jest-mock": "29.7.0", + "jest": "30.2.0", + "jest-environment-node": "30.2.0", + "jest-mock": "30.2.0", "nodemon": "3.1.10", - "pid-port": "1.0.2", + "pid-port": "2.0.0", "simple-oauth2": "5.1.0", - "supertest": "7.1.0" + "supertest": "7.1.4", + "ts-jest": "29.4.5", + "ts-node": "10.9.2", + "typescript": "5.9.2" } } diff --git a/packages/backend/scripts/check_connect.js b/packages/backend/scripts/check_connect.js index 137712660a..3a8f1c4abb 100644 --- a/packages/backend/scripts/check_connect.js +++ b/packages/backend/scripts/check_connect.js @@ -6,14 +6,18 @@ import Redis from 'ioredis'; import { loadConfig } from '../built/config.js'; import { createPostgresDataSource } from '../built/postgres.js'; +import { LoggerService } from '../built/core/LoggerService.js'; +import { NativeTimeService } from '../built/global/TimeService.js'; +import { EnvService } from '../built/global/EnvService.js'; -const config = loadConfig(); +const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService()); +const config = loadConfig(loggerService); // createPostgresDataSource handles primaries and replicas automatically. // usually, it only opens connections first use, so we force it using // .initialize() -async function connectToPostgres(){ - const source = createPostgresDataSource(config); +async function connectToPostgres() { + const source = createPostgresDataSource(config, loggerService); await source.initialize(); await source.destroy(); } @@ -32,10 +36,8 @@ async function connectToRedis(redisOptions) { try { await redis.connect(); resolve(); - } catch (e) { reject(e); - } finally { redis.disconnect(false); } diff --git a/packages/backend/scripts/dev.mjs b/packages/backend/scripts/dev.mjs index a3e0558abd..05a8d01ffc 100644 --- a/packages/backend/scripts/dev.mjs +++ b/packages/backend/scripts/dev.mjs @@ -5,7 +5,7 @@ import { execa, execaNode } from 'execa'; -/** @type {import('execa').ExecaChildProcess | undefined} */ +/** @type {import('execa').ResultPromise | undefined} */ let backendProcess; async function execBuildAssets() { @@ -13,7 +13,7 @@ async function execBuildAssets() { cwd: '../../', stdout: process.stdout, stderr: process.stderr, - }) + }); } function execStart() { @@ -32,8 +32,8 @@ async function killProc() { if (backendProcess) { backendProcess.catch(() => {}); // backendProcess.kill()によって発生する例外を無視するためにcatch()を呼び出す backendProcess.kill(); - await new Promise(resolve => backendProcess.on('exit', resolve)); - backendProcess = undefined; + await new Promise(resolve => backendProcess?.on('exit', resolve)) + .finally(() => backendProcess = undefined); } } @@ -49,7 +49,7 @@ async function killProc() { stdio: [process.stdin, process.stdout, process.stderr, 'ipc'], serialization: "json", }) - .on('message', async (message) => { + .on('message', /** @param {{type: string}} message */ async (message) => { if (message.type === 'exit') { // かならずbuild->build-assetsの順番で呼び出したいので、 // 少々トリッキーだがnodemonからのexitイベントを利用してbuild-assets->startを行う。 @@ -59,5 +59,5 @@ async function killProc() { await execBuildAssets(); execStart(); } - }) + }); })(); diff --git a/packages/backend/scripts/generate_api_json.js b/packages/backend/scripts/generate_api_json.js index 798e243004..c554755e74 100644 --- a/packages/backend/scripts/generate_api_json.js +++ b/packages/backend/scripts/generate_api_json.js @@ -5,6 +5,9 @@ import { execa } from 'execa'; import { writeFileSync, existsSync } from "node:fs"; +import { LoggerService } from '../built/core/LoggerService.js'; +import { NativeTimeService } from '../built/global/TimeService.js'; +import { EnvService } from '../built/global/EnvService.js'; async function main() { if (!process.argv.includes('--no-build')) { @@ -24,7 +27,8 @@ async function main() { /** @type {import('../src/server/api/openapi/gen-spec.js')} */ const { genOpenapiSpec } = await import('../built/server/api/openapi/gen-spec.js'); - const config = loadConfig(); + const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService()); + const config = loadConfig(loggerService); const spec = genOpenapiSpec(config, true); writeFileSync('./built/api.json', JSON.stringify(spec), 'utf-8'); diff --git a/packages/backend/scripts/watch.mjs b/packages/backend/scripts/watch.mjs index a0ccea3b16..cd92eb0701 100644 --- a/packages/backend/scripts/watch.mjs +++ b/packages/backend/scripts/watch.mjs @@ -8,20 +8,20 @@ import { execa } from 'execa'; (async () => { // なぜかchokidarが動かない影響で、watchされない /* - execa('tsc-alias', ['-w', '-p', 'tsconfig.json'], { + execa('tsc-alias', ['-w', '-p', 'tsconfig.backend.json'], { stdout: process.stdout, stderr: process.stderr, }); */ setInterval(() => { - execa('tsc-alias', ['-p', 'tsconfig.json'], { + execa('tsc-alias', ['-p', 'tsconfig.backend.json'], { stdout: process.stdout, stderr: process.stderr, }); }, 3000); - execa('tsc', ['-w', '-p', 'tsconfig.json'], { + execa('tsc', ['-w', '-p', 'tsconfig.backend.json'], { stdout: process.stdout, stderr: process.stderr, }); diff --git a/packages/backend/src/@types/redis-lock.d.ts b/packages/backend/src/@types/redis-lock.d.ts index b037cde5ee..e27b943956 100644 --- a/packages/backend/src/@types/redis-lock.d.ts +++ b/packages/backend/src/@types/redis-lock.d.ts @@ -4,9 +4,14 @@ */ declare module 'redis-lock' { - import type Redis from 'ioredis'; + export interface NodeRedis { + readonly v4: true; + set(key: string, value: string | number, opts?: { PX?: number, NX?: boolean }): Promise<'OK' | null>; + del(key: string): Promise; + } - type Lock = (lockName: string, timeout?: number, taskToPerform?: () => Promise) => void; + export type Unlock = () => Promise; + export type Lock = (lockName: string, timeout?: number) => Promise; function redisLock(client: Redis.Redis, retryDelay: number): Lock; export = redisLock; diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index d4ff7b7f84..e6c84b544b 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -8,6 +8,14 @@ import * as Redis from 'ioredis'; import { DataSource } from 'typeorm'; import { MeiliSearch } from 'meilisearch'; import { MiMeta } from '@/models/Meta.js'; +import { bindThis } from '@/decorators.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; +import { TimeService, NativeTimeService } from '@/global/TimeService.js'; +import { EnvService } from '@/global/EnvService.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { DependencyService } from '@/global/DependencyService.js'; +import { LoggerService } from '@/core/LoggerService.js'; import { DI } from './di-symbols.js'; import { Config, loadConfig } from './config.js'; import { createPostgresDataSource } from './postgres.js'; @@ -19,21 +27,17 @@ import type { Provider, OnApplicationShutdown } from '@nestjs/common'; const $config: Provider = { provide: DI.config, - useValue: loadConfig(), + useFactory: (loggerService: LoggerService) => loadConfig(loggerService), + inject: [LoggerService], }; const $db: Provider = { provide: DI.db, - useFactory: async (config) => { - try { - const db = createPostgresDataSource(config); - return await db.initialize(); - } catch (e) { - console.error('failed to initialize database connection', e); - throw e; - } + useFactory: async (config: Config, loggerService: LoggerService) => { + const db = createPostgresDataSource(config, loggerService); + return await db.initialize(); }, - inject: [DI.config], + inject: [DI.config, LoggerService], }; const $meilisearch: Provider = { @@ -163,11 +167,22 @@ const $meta: Provider = { inject: [DI.db, DI.redisForSub], }; +const $CacheManagementService: Provider[] = [CacheManagementService, { provide: 'CacheManagementService', useExisting: CacheManagementService }]; +const $InternalEventService: Provider[] = [InternalEventService, { provide: 'InternalEventService', useExisting: InternalEventService }]; +const $TimeService: Provider[] = [ + { provide: TimeService, useClass: NativeTimeService }, + { provide: 'TimeService', useExisting: TimeService }, +]; +const $EnvService: Provider[] = [EnvService, { provide: 'EnvService', useExisting: EnvService }]; +const $LoggerService: Provider[] = [LoggerService, { provide: 'LoggerService', useExisting: LoggerService }]; +const $Console: Provider[] = [{ provide: DI.console, useFactory: () => global.console }]; // useValue will break overrideProvider for some reason +const $DependencyService: Provider[] = [DependencyService, { provide: 'DependencyService', useExisting: DependencyService }]; + @Global() @Module({ imports: [RepositoryModule], - providers: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit], - exports: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit, RepositoryModule], + providers: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit, $CacheManagementService, $InternalEventService, $TimeService, $EnvService, $LoggerService, $Console, $DependencyService].flat(), + exports: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit, $CacheManagementService, $InternalEventService, $TimeService, $EnvService, $LoggerService, RepositoryModule, $Console, $DependencyService].flat(), }) export class GlobalModule implements OnApplicationShutdown { private readonly logger = new Logger('global'); @@ -189,16 +204,25 @@ export class GlobalModule implements OnApplicationShutdown { // And then disconnect from DB this.logger.info('Disconnected from data sources...'); await this.db.destroy(); - this.redisClient.disconnect(); - this.redisForPub.disconnect(); - this.redisForSub.disconnect(); - this.redisForTimelines.disconnect(); - this.redisForReactions.disconnect(); - this.redisForRateLimit.disconnect(); + this.safeDisconnect(this.redisClient); + this.safeDisconnect(this.redisForPub); + this.safeDisconnect(this.redisForSub); + this.safeDisconnect(this.redisForTimelines); + this.safeDisconnect(this.redisForReactions); + this.safeDisconnect(this.redisForRateLimit); this.logger.info('Global module disposed.'); } + @bindThis async onApplicationShutdown(signal: string): Promise { await this.dispose(); } + + private safeDisconnect(redis: { disconnect(): void }): void { + try { + redis.disconnect(); + } catch (err) { + this.logger.error(`Unhandled error disconnecting redis: ${renderInlineError(err)}`); + } + } } diff --git a/packages/backend/src/MainModule.ts b/packages/backend/src/MainModule.ts index f86a0be93c..0e8df2a928 100644 --- a/packages/backend/src/MainModule.ts +++ b/packages/backend/src/MainModule.ts @@ -5,12 +5,10 @@ import { Module } from '@nestjs/common'; import { ServerModule } from '@/server/ServerModule.js'; -import { GlobalModule } from '@/GlobalModule.js'; import { DaemonModule } from '@/daemons/DaemonModule.js'; @Module({ imports: [ - GlobalModule, ServerModule, DaemonModule, ], diff --git a/packages/backend/src/NestLogger.ts b/packages/backend/src/NestLogger.ts index d0be19664f..d011c5e42e 100644 --- a/packages/backend/src/NestLogger.ts +++ b/packages/backend/src/NestLogger.ts @@ -4,10 +4,9 @@ */ import { LoggerService } from '@nestjs/common'; -import Logger from '@/logger.js'; +import { coreLogger } from '@/boot/coreLogger.js'; -const logger = new Logger('core', 'cyan'); -const nestLogger = logger.createSubLogger('nest', 'green'); +const nestLogger = coreLogger.createSubLogger('nest', 'green'); export class NestLogger implements LoggerService { /** diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts index a10597c7a7..8ba25c7c0b 100644 --- a/packages/backend/src/boot/common.ts +++ b/packages/backend/src/boot/common.ts @@ -12,7 +12,7 @@ import { QueueStatsService } from '@/daemons/QueueStatsService.js'; import { ServerStatsService } from '@/daemons/ServerStatsService.js'; import { ServerService } from '@/server/ServerService.js'; import { MainModule } from '@/MainModule.js'; -import { envOption } from '@/env.js'; +import { EnvService } from '@/global/EnvService.js'; import { ApLogCleanupService } from '@/daemons/ApLogCleanupService.js'; export async function server() { @@ -27,7 +27,9 @@ export async function server() { if (process.env.NODE_ENV !== 'test') { app.get(ChartManagementService).start(); } - if (!envOption.noDaemons) { + + const envService = app.get(EnvService); + if (!envService.options.noDaemons) { app.get(QueueStatsService).start(); app.get(ServerStatsService).start(); app.get(ApLogCleanupService).start(); diff --git a/packages/backend/src/boot/coreLogger.ts b/packages/backend/src/boot/coreLogger.ts new file mode 100644 index 0000000000..32fc406758 --- /dev/null +++ b/packages/backend/src/boot/coreLogger.ts @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { EnvService } from '@/global/EnvService.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import { NativeTimeService } from '@/global/TimeService.js'; + +export const coreEnvService = new EnvService(); + +// eslint-disable-next-line no-restricted-globals +export const coreLoggerService = new LoggerService(console, new NativeTimeService(), coreEnvService); +export const coreLogger = coreLoggerService.getLogger('core', 'cyan'); diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index bbe6a57383..dc7d9cf054 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -8,31 +8,28 @@ */ import cluster from 'node:cluster'; -import { EventEmitter } from 'node:events'; -import { inspect } from 'node:util'; import chalk from 'chalk'; import Xev from 'xev'; -import Logger from '@/logger.js'; -import { envOption } from '../env.js'; +import { coreLogger, coreEnvService, coreLoggerService } from '@/boot/coreLogger.js'; +import { prepEnv } from '@/boot/prepEnv.js'; import { masterMain } from './master.js'; import { workerMain } from './worker.js'; import { readyRef } from './ready.js'; -import 'reflect-metadata'; - process.title = `Misskey (${cluster.isPrimary ? 'master' : 'worker'})`; -Error.stackTraceLimit = Infinity; -EventEmitter.defaultMaxListeners = 128; +prepEnv(); -const logger = new Logger('core', 'cyan'); -const clusterLogger = logger.createSubLogger('cluster', 'orange'); const ev = new Xev(); // We wrap this in a main function, that gets called, // because not all platforms support top level await :/ async function main() { + const envOption = coreEnvService.options; + const clusterLogger = coreLogger.createSubLogger('cluster', 'orange'); + const logger = coreLogger; + //#region Events // Listen new workers cluster.on('fork', worker => { @@ -52,36 +49,6 @@ async function main() { cluster.fork(); }); - // Display detail of unhandled promise rejection - if (!envOption.quiet) { - process.on('unhandledRejection', e => { - logger.error('Unhandled rejection:', inspect(e)); - }); - } - - process.on('uncaughtException', (err) => { - // Workaround for https://github.com/node-fetch/node-fetch/issues/954 - if (String(err).match(/^TypeError: .+ is an? url with embedded credentials.$/)) { - logger.debug('Suppressed node-fetch issue#954, but the current job may fail.'); - return; - } - - // Workaround for https://github.com/node-fetch/node-fetch/issues/1845 - if (String(err) === 'TypeError: Cannot read properties of undefined (reading \'body\')') { - logger.debug('Suppressed node-fetch issue#1845, but the current job may fail.'); - return; - } - - // Throw all other errors to avoid inconsistent state. - // (per NodeJS docs, it's unsafe to suppress arbitrary errors in an uncaughtException handler.) - throw err; - }); - - // Display detail of uncaught exception - process.on('uncaughtExceptionMonitor', (err, origin) => { - logger.error(`Uncaught exception (${origin}):`, err); - }); - // Dying away... process.on('disconnect', () => { logger.warn('IPC channel disconnected! The process may soon die.'); @@ -97,18 +64,18 @@ async function main() { if (!envOption.disableClustering) { if (cluster.isPrimary) { logger.info(`Start main process... pid: ${process.pid}`); - await masterMain(); + await masterMain(coreLoggerService, coreEnvService); ev.mount(); } else if (cluster.isWorker) { logger.info(`Start worker process... pid: ${process.pid}`); - await workerMain(); + await workerMain(coreLoggerService, coreEnvService); } else { throw new Error('Unknown process type'); } } else { // 非clusterの場合はMasterのみが起動するため、Workerの処理は行わない(cluster.isWorker === trueの状態でこのブロックに来ることはない) logger.info(`Start main process... pid: ${process.pid}`); - await masterMain(); + await masterMain(coreLoggerService, coreEnvService); ev.mount(); } diff --git a/packages/backend/src/boot/master.ts b/packages/backend/src/boot/master.ts index 607e8de340..3f8acaec30 100644 --- a/packages/backend/src/boot/master.ts +++ b/packages/backend/src/boot/master.ts @@ -13,11 +13,15 @@ import chalk from 'chalk'; import chalkTemplate from 'chalk-template'; import * as Sentry from '@sentry/node'; import { nodeProfilingIntegration } from '@sentry/profiling-node'; -import Logger from '@/logger.js'; +import type Logger from '@/logger.js'; import { loadConfig } from '@/config.js'; import type { Config } from '@/config.js'; +import type { LoggerService } from '@/core/LoggerService.js'; +import type { EnvService } from '@/global/EnvService.js'; +import type { EnvOption } from '@/env.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { showMachineInfo } from '@/misc/show-machine-info.js'; -import { envOption } from '@/env.js'; +import { coreLogger } from '@/boot/coreLogger.js'; import { jobQueue, server } from './common.js'; const _filename = fileURLToPath(import.meta.url); @@ -25,29 +29,26 @@ const _dirname = dirname(_filename); const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json`, 'utf-8')); -const logger = new Logger('core', 'cyan'); -const bootLogger = logger.createSubLogger('boot', 'magenta'); - const themeColor = chalk.hex('#86b300'); -function greet() { +function greet(logger: Logger, bootLogger: Logger, envOption: EnvOption) { if (!envOption.quiet) { //#region Misskey logo - console.log(themeColor(' _____ _ _ ')); - console.log(themeColor('/ ___| | | | ')); - console.log(themeColor('\\ `--.| |__ __ _ _ __| | _____ _ _ ')); - console.log(themeColor(' `--. \\ \'_ \\ / _` | \'__| |/ / _ \\ | | |')); - console.log(themeColor('/\\__/ / | | | (_| | | | < __/ |_| |')); - console.log(themeColor('\\____/|_| |_|\\__,_|_| |_|\\_\\___|\\__, |')); - console.log(themeColor(' __/ |')); - console.log(themeColor(' |___/ ')); + logger.info(themeColor(' _____ _ _ ')); + logger.info(themeColor('/ ___| | | | ')); + logger.info(themeColor('\\ `--.| |__ __ _ _ __| | _____ _ _ ')); + logger.info(themeColor(' `--. \\ \'_ \\ / _` | \'__| |/ / _ \\ | | |')); + logger.info(themeColor('/\\__/ / | | | (_| | | | < __/ |_| |')); + logger.info(themeColor('\\____/|_| |_|\\__,_|_| |_|\\_\\___|\\__, |')); + logger.info(themeColor(' __/ |')); + logger.info(themeColor(' |___/ ')); //#endregion - console.log(' Sharkey is an open-source decentralized microblogging platform.'); - console.log(chalk.rgb(255, 136, 0)(' If you like Sharkey, please donate to support development. https://opencollective.com/sharkey')); + logger.info(' Sharkey is an open-source decentralized microblogging platform.'); + logger.info(chalk.rgb(255, 136, 0)(' If you like Sharkey, please donate to support development. https://opencollective.com/sharkey')); - console.log(''); - console.log(chalkTemplate`--- ${os.hostname()} {gray (PID: ${process.pid.toString()})} ---`); + logger.info(''); + logger.info(chalkTemplate`--- ${os.hostname()} {gray (PID: ${process.pid.toString()})} ---`); } bootLogger.info('Welcome to Sharkey!'); @@ -57,20 +58,23 @@ function greet() { /** * Init master process */ -export async function masterMain() { +export async function masterMain(loggerService: LoggerService, envService: EnvService) { let config!: Config; + const bootLogger = coreLogger.createSubLogger('boot', 'magenta'); + const envOption = envService.options; + // initialize app try { - greet(); - showEnvironment(); + greet(coreLogger, bootLogger, envOption); + showEnvironment(bootLogger); await showMachineInfo(bootLogger); - showNodejsVersion(); - config = loadConfigBoot(); + showNodejsVersion(bootLogger); + config = loadConfig(loggerService); //await connectDb(); if (config.pidFile) fs.writeFileSync(config.pidFile, process.pid.toString()); } catch (e) { - bootLogger.error('Fatal error occurred during initialization', null, true); + bootLogger.error(`Fatal error occurred during initialization: ${renderInlineError(e)}`, { e }, true); process.exit(1); } @@ -125,7 +129,7 @@ export async function masterMain() { process.exit(1); } - await spawnWorkers(config.clusterLimit); + await spawnWorkers(bootLogger, config.clusterLimit); } else { // clusterモジュール無効時 @@ -147,7 +151,7 @@ export async function masterMain() { } } -function showEnvironment(): void { +function showEnvironment(bootLogger: Logger): void { const env = process.env.NODE_ENV; const logger = bootLogger.createSubLogger('env'); logger.info(typeof env === 'undefined' ? 'NODE_ENV is not set' : `NODE_ENV: ${env}`); @@ -158,34 +162,12 @@ function showEnvironment(): void { } } -function showNodejsVersion(): void { +function showNodejsVersion(bootLogger: Logger): void { const nodejsLogger = bootLogger.createSubLogger('nodejs'); nodejsLogger.info(`Version ${process.version} detected.`); } -function loadConfigBoot(): Config { - const configLogger = bootLogger.createSubLogger('config'); - let config; - - try { - config = loadConfig(); - } catch (exception) { - if (typeof exception === 'string') { - configLogger.error('Exception loading config:', exception); - process.exit(1); - } else if ((exception as any).code === 'ENOENT') { - configLogger.error('Configuration file not found', null, true); - process.exit(1); - } - throw exception; - } - - configLogger.info('Loaded'); - - return config; -} - /* async function connectDb(): Promise { const dbLogger = bootLogger.createSubLogger('db'); @@ -204,17 +186,17 @@ async function connectDb(): Promise { } */ -async function spawnWorkers(limit = 1) { +async function spawnWorkers(bootLogger: Logger, limit = 1) { const cpuCount = os.cpus().length; // in some weird environments, node can't count the CPUs; we trust the config in those cases const workers = cpuCount === 0 ? limit : Math.min(limit, cpuCount); bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`); - await Promise.all([...Array(workers)].map(spawnWorker)); + await Promise.all([...Array(workers)].map(() => spawnWorker(bootLogger))); bootLogger.info('All workers started'); } -function spawnWorker(): Promise { +function spawnWorker(bootLogger: Logger): Promise { return new Promise(res => { const worker = cluster.fork(); worker.on('message', message => { diff --git a/packages/backend/src/boot/prepEnv.ts b/packages/backend/src/boot/prepEnv.ts new file mode 100644 index 0000000000..271352eec9 --- /dev/null +++ b/packages/backend/src/boot/prepEnv.ts @@ -0,0 +1,56 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { EventEmitter } from 'node:events'; +import { inspect } from 'node:util'; +import { coreLogger } from '@/boot/coreLogger.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; + +// Polyfill reflection metadata *without* loading dependencies that may corrupt native types. +// https://github.com/microsoft/reflect-metadata?tab=readme-ov-file#es-modules-in-nodejsbrowser-typescriptbabel-bundlers +import 'reflect-metadata/lite'; + +/** + * Configures Node.JS global runtime options for values appropriate for Sharkey. + */ +export function prepEnv() { + // Increase maximum stack trace length. + // This helps diagnose infinite recursion bugs. + Error.stackTraceLimit = Infinity; + + // Avoid warnings like "11 message listeners added to [Commander]. MaxListeners is 10." + // This is expected due to use of NestJS lifecycle hooks. + EventEmitter.defaultMaxListeners = 128; + + // Workaround certain 3rd-party bugs + process.on('uncaughtException', (err) => { + // Workaround for https://github.com/node-fetch/node-fetch/issues/954 + if (String(err).match(/^TypeError: .+ is an? url with embedded credentials.$/)) { + coreLogger.debug('Suppressed node-fetch issue#954, but the current job may fail.'); + return; + } + + // Workaround for https://github.com/node-fetch/node-fetch/issues/1845 + if (String(err) === 'TypeError: Cannot read properties of undefined (reading \'body\')') { + coreLogger.debug('Suppressed node-fetch issue#1845, but the current job may fail.'); + return; + } + + // Throw all other errors to avoid inconsistent state. + // (per NodeJS docs, it's unsafe to suppress arbitrary errors in an uncaughtException handler.) + coreLogger.error(`Uncaught exception: ${renderInlineError(err)}`, { + error: inspect(err), + }); + throw err; + }); + + // Log uncaught promise rejections + process.on('unhandledRejection', (error, promise) => { + coreLogger.error(`Unhandled rejection: ${renderInlineError(error)}`, { + error: inspect(error), + promise: inspect(promise), + }); + }); +} diff --git a/packages/backend/src/boot/worker.ts b/packages/backend/src/boot/worker.ts index 8cf3cadd22..762471e88b 100644 --- a/packages/backend/src/boot/worker.ts +++ b/packages/backend/src/boot/worker.ts @@ -4,14 +4,16 @@ */ import cluster from 'node:cluster'; -import * as Sentry from '@sentry/node'; -import { nodeProfilingIntegration } from '@sentry/profiling-node'; -import { envOption } from '@/env.js'; -import { loadConfig } from '@/config.js'; -import { jobQueue, server } from './common.js'; import { fileURLToPath } from 'node:url'; import { dirname } from 'node:path'; import * as fs from 'node:fs'; +import * as Sentry from '@sentry/node'; +import { nodeProfilingIntegration } from '@sentry/profiling-node'; +import { loadConfig } from '@/config.js'; +import type { LoggerService } from '@/core/LoggerService.js'; +import type { EnvService } from '@/global/EnvService.js'; +import { jobQueue, server } from './common.js'; + const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json`, 'utf-8')); @@ -19,8 +21,9 @@ const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json /** * Init worker process */ -export async function workerMain() { - const config = loadConfig(); +export async function workerMain(loggerService: LoggerService, envService: EnvService) { + const config = loadConfig(loggerService); + const envOption = envService.options; if (config.sentryForBackend) { Sentry.init({ @@ -37,7 +40,7 @@ export async function workerMain() { maxBreadcrumbs: 0, // Set release version - release: "Sharkey@" + (meta.gitVersion ?? meta.version), + release: 'Sharkey@' + (meta.gitVersion ?? meta.version), ...config.sentryForBackend.options, }); diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 102cb69dcb..5607f50eb7 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -14,6 +14,7 @@ import type * as Sentry from '@sentry/node'; import type * as SentryVue from '@sentry/vue'; import type { RedisOptions } from 'ioredis'; import type { IPv4, IPv6 } from 'ipaddr.js'; +import type { LoggerService } from '@/core/LoggerService.js'; type RedisOptionsSource = Partial & { host?: string; @@ -144,7 +145,6 @@ type Source = { disableQueryTruncation?: boolean, enableQueryParamLogging?: boolean, }; - verbose?: boolean; } activityLogging?: { @@ -160,8 +160,6 @@ type Source = { } }; -const configLogger = new Logger('config'); - export type PrivateNetworkSource = string | { network?: string, ports?: number[] }; export type PrivateNetwork = { @@ -180,10 +178,10 @@ export type PrivateNetwork = { export type CIDR = [ip: IPv4 | IPv6, prefixLength: number]; -export function parsePrivateNetworks(patterns: PrivateNetworkSource[]): PrivateNetwork[]; -export function parsePrivateNetworks(patterns: undefined): undefined; -export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined): PrivateNetwork[] | undefined; -export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined): PrivateNetwork[] | undefined { +export function parsePrivateNetworks(patterns: PrivateNetworkSource[], configLogger: Logger): PrivateNetwork[]; +export function parsePrivateNetworks(patterns: undefined, configLogger: Logger): undefined; +export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined, configLogger: Logger): PrivateNetwork[] | undefined; +export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined, configLogger: Logger): PrivateNetwork[] | undefined { if (!patterns) return undefined; return patterns .map(e => { @@ -290,7 +288,6 @@ export type Config = { disableQueryTruncation?: boolean, enableQueryParamLogging?: boolean, }; - verbose?: boolean; } version: string; @@ -370,7 +367,9 @@ const path = process.env.MISSKEY_CONFIG_YML ? resolve(dir, 'test.yml') : resolve(dir, 'default.yml'); -export function loadConfig(): Config { +export function loadConfig(loggerService: LoggerService): Config { + const configLogger = loggerService.getLogger('config'); + const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8')); const frontendManifestExists = fs.existsSync(_dirname + '/../../../built/_frontend_vite_/manifest.json'); @@ -458,7 +457,7 @@ export function loadConfig(): Config { proxy: config.proxy, proxySmtp: config.proxySmtp, proxyBypassHosts: config.proxyBypassHosts, - allowedPrivateNetworks: parsePrivateNetworks(config.allowedPrivateNetworks), + allowedPrivateNetworks: parsePrivateNetworks(config.allowedPrivateNetworks, configLogger), disallowExternalApRedirect: config.disallowExternalApRedirect ?? false, maxFileSize: config.maxFileSize ?? 262144000, maxNoteLength: config.maxNoteLength ?? 3000, @@ -671,7 +670,6 @@ function applyEnvOverrides(config: Source) { _apply_top(['import', ['downloadTimeout', 'maxFileSize']]); _apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword', 'disallowExternalApRedirect']]); _apply_top(['logging', 'sql', ['disableQueryTruncation', 'enableQueryParamLogging']]); - _apply_top(['logging', ['verbose']]); _apply_top(['activityLogging', ['enabled', 'preSave', 'maxAge']]); _apply_top(['customHtml', ['head']]); } diff --git a/packages/backend/src/core/AbuseReportNotificationService.ts b/packages/backend/src/core/AbuseReportNotificationService.ts index 307f22586e..23a0fa1c7b 100644 --- a/packages/backend/src/core/AbuseReportNotificationService.ts +++ b/packages/backend/src/core/AbuseReportNotificationService.ts @@ -23,6 +23,7 @@ import { RecipientMethod } from '@/models/AbuseReportNotificationRecipient.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { SystemWebhookService } from '@/core/SystemWebhookService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { TimeService } from '@/global/TimeService.js'; import { IdService } from './IdService.js'; @Injectable() @@ -44,6 +45,7 @@ export class AbuseReportNotificationService implements OnApplicationShutdown { private moderationLogService: ModerationLogService, private globalEventService: GlobalEventService, private userEntityService: UserEntityService, + private readonly timeService: TimeService, ) { this.redisForSub.on('message', this.onMessage); } @@ -326,7 +328,7 @@ export class AbuseReportNotificationService implements OnApplicationShutdown { await this.abuseReportNotificationRecipientRepository.update(params.id, { isActive: params.isActive, - updatedAt: new Date(), + updatedAt: this.timeService.date, name: params.name, method: params.method, userId: params.userId, diff --git a/packages/backend/src/core/AccountMoveService.ts b/packages/backend/src/core/AccountMoveService.ts index e107f02796..82e68e32aa 100644 --- a/packages/backend/src/core/AccountMoveService.ts +++ b/packages/backend/src/core/AccountMoveService.ts @@ -27,6 +27,8 @@ import { SystemAccountService } from '@/core/SystemAccountService.js'; import { RoleService } from '@/core/RoleService.js'; import { AntennaService } from '@/core/AntennaService.js'; import { CacheService } from '@/core/CacheService.js'; +import { UserListService } from '@/core/UserListService.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class AccountMoveService { @@ -70,6 +72,8 @@ export class AccountMoveService { private roleService: RoleService, private antennaService: AntennaService, private readonly cacheService: CacheService, + private readonly userListService: UserListService, + private readonly timeService: TimeService, ) { } @@ -87,7 +91,7 @@ export class AccountMoveService { const update = {} as Partial; update.alsoKnownAs = src.alsoKnownAs?.includes(dstUri) ? src.alsoKnownAs : src.alsoKnownAs?.concat([dstUri]) ?? [dstUri]; update.movedToUri = dstUri; - update.movedAt = new Date(); + update.movedAt = this.timeService.date; await this.usersRepository.update(src.id, update); Object.assign(src, update); @@ -179,7 +183,7 @@ export class AccountMoveService { // Insert new mutings with the same values except mutee const oldMutings = await this.mutingsRepository.findBy([ { muteeId: src.id, expiresAt: IsNull() }, - { muteeId: src.id, expiresAt: MoreThan(new Date()) }, + { muteeId: src.id, expiresAt: MoreThan(this.timeService.date) }, ]); if (oldMutings.length === 0) return; @@ -263,45 +267,35 @@ export class AccountMoveService { @bindThis public async updateLists(src: ThinUser, dst: MiUser): Promise { // Return if there is no list to be updated. - const oldMemberships = await this.userListMembershipsRepository.find({ - where: { - userId: src.id, - }, - }); - if (oldMemberships.length === 0) return; + const [srcMemberships, dstMemberships] = await Promise.all([ + this.cacheService.userListMembershipsCache.fetch(src.id), + this.cacheService.userListMembershipsCache.fetch(dst.id), + ]); + if (srcMemberships.size === 0) return; - const existingUserListIds = await this.userListMembershipsRepository.find({ - where: { - userId: dst.id, - }, - }).then(memberships => memberships.map(membership => membership.userListId)); + const newMemberships = srcMemberships.values() + .filter(srcMembership => !dstMemberships.has(srcMembership.userListId)) + .map(srcMembership => ({ + userListId: srcMembership.userListId, + withReplies: srcMembership.withReplies, + })) + .toArray(); + const updatedMemberships = srcMemberships.values() + .filter(srcMembership => { + const dstMembership = dstMemberships.get(srcMembership.userListId); + return dstMembership != null && dstMembership.withReplies !== srcMembership.withReplies; + }) + .map(srcMembership => ({ + userListId: srcMembership.userListId, + withReplies: srcMembership.withReplies, + })) + .toArray(); - const newMemberships: Map = new Map(); - - // 重複しないようにIDを生成 - const genId = (): string => { - let id: string; - do { - id = this.idService.gen(); - } while (newMemberships.has(id)); - return id; - }; - for (const membership of oldMemberships) { - if (existingUserListIds.includes(membership.userListId)) continue; // skip if dst exists in this user's list - newMemberships.set(genId(), { - userId: dst.id, - userListId: membership.userListId, - userListUserId: membership.userListUserId, - }); + if (newMemberships.length > 0) { + await this.userListService.bulkAddMember(dst, newMemberships); } - - const arrayToInsert = Array.from(newMemberships.entries()).map(entry => ({ ...entry[1], id: entry[0] })); - await this.userListMembershipsRepository.insert(arrayToInsert); - - // Have the proxy account follow the new account in the same way as UserListService.push - if (this.userEntityService.isRemoteUser(dst)) { - const proxy = await this.systemAccountService.fetch('proxy'); - this.queueService.createFollowJob([{ from: { id: proxy.id }, to: { id: dst.id } }]); + if (updatedMemberships.length > 0) { + await this.userListService.bulkUpdateMembership(dst, updatedMemberships); } } @@ -356,7 +350,7 @@ export class AccountMoveService { let resultUser: MiLocalUser | MiRemoteUser | null = null; if (this.userEntityService.isRemoteUser(dst)) { - if (Date.now() - (dst.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) { + if (this.timeService.now - (dst.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) { await this.apPersonService.updatePerson(dst.uri); } dst = await this.apPersonService.fetchPerson(dst.uri) ?? dst; @@ -372,7 +366,7 @@ export class AccountMoveService { if (!src) continue; // oldAccountを探してもこのサーバーに存在しない場合はフォロー関係もないということなのでスルー if (this.userEntityService.isRemoteUser(dst)) { - if (Date.now() - (src.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) { + if (this.timeService.now - (src.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) { await this.apPersonService.updatePerson(srcUri); } diff --git a/packages/backend/src/core/AchievementService.ts b/packages/backend/src/core/AchievementService.ts index 8d2de89efd..b961fac991 100644 --- a/packages/backend/src/core/AchievementService.ts +++ b/packages/backend/src/core/AchievementService.ts @@ -9,6 +9,7 @@ import type { MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; import { ACHIEVEMENT_TYPES } from '@/models/UserProfile.js'; @Injectable() @@ -18,6 +19,7 @@ export class AchievementService { private userProfilesRepository: UserProfilesRepository, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { } @@ -28,7 +30,7 @@ export class AchievementService { ): Promise { if (!ACHIEVEMENT_TYPES.includes(type)) return; - const date = Date.now(); + const date = this.timeService.now; const profile = await this.userProfilesRepository.findOneByOrFail({ userId: userId }); diff --git a/packages/backend/src/core/AnnouncementService.ts b/packages/backend/src/core/AnnouncementService.ts index ddeea1eed6..54496f9922 100644 --- a/packages/backend/src/core/AnnouncementService.ts +++ b/packages/backend/src/core/AnnouncementService.ts @@ -17,6 +17,7 @@ import { ModerationLogService } from '@/core/ModerationLogService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { Config } from '@/config.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class AnnouncementService { @@ -38,6 +39,7 @@ export class AnnouncementService { private moderationLogService: ModerationLogService, private announcementEntityService: AnnouncementEntityService, private roleService: RoleService, + private readonly timeService: TimeService, ) { } @@ -143,7 +145,7 @@ export class AnnouncementService { } await this.announcementsRepository.update(announcement.id, { - updatedAt: new Date(), + updatedAt: this.timeService.date, title: values.title, text: values.text, /* eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- 空の文字列の場合、nullを渡すようにするため */ diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 667df57943..660c97dd6a 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -139,12 +139,8 @@ export class AntennaService implements OnApplicationShutdown { // TODO } else if (antenna.src === 'list') { if (antenna.userListId == null) return false; - const exists = await this.userListMembershipsRepository.exists({ - where: { - userListId: antenna.userListId, - userId: note.userId, - }, - }); + const memberships = await this.cacheService.userListMembershipsCache.fetch(note.userId); + const exists = memberships.has(antenna.userListId); if (!exists) return false; } else if (antenna.src === 'users') { const accts = antenna.users.map(x => { diff --git a/packages/backend/src/core/ApLogService.ts b/packages/backend/src/core/ApLogService.ts index f21c6da313..89837a60d2 100644 --- a/packages/backend/src/core/ApLogService.ts +++ b/packages/backend/src/core/ApLogService.ts @@ -12,6 +12,7 @@ import type { ApContextsRepository, ApFetchLogsRepository, ApInboxLogsRepository import type { Config } from '@/config.js'; import { JsonValue } from '@/misc/json-value.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { TimeService } from '@/global/TimeService.js'; import { IdService } from '@/core/IdService.js'; import { IActivity, IObject } from './activitypub/type.js'; @@ -32,6 +33,7 @@ export class ApLogService { private readonly utilityService: UtilityService, private readonly idService: IdService, + private readonly timeService: TimeService, ) {} /** @@ -46,7 +48,7 @@ export class ApLogService { const log = new SkApInboxLog({ id: this.idService.gen(), - at: new Date(), + at: this.timeService.date, verified: false, accepted: false, host, @@ -85,7 +87,7 @@ export class ApLogService { }): Promise { const log = new SkApFetchLog({ id: this.idService.gen(), - at: new Date(), + at: this.timeService.date, accepted: false, ...data, }); @@ -163,7 +165,7 @@ export class ApLogService { */ public async deleteExpiredLogs(): Promise { // This is the date in UTC of the oldest log to KEEP - const oldestAllowed = new Date(Date.now() - this.config.activityLogging.maxAge); + const oldestAllowed = new Date(this.timeService.now - this.config.activityLogging.maxAge); // Delete all logs older than the threshold. const inboxDeleted = await this.deleteExpiredInboxLogs(oldestAllowed); diff --git a/packages/backend/src/core/AppLockService.ts b/packages/backend/src/core/AppLockService.ts index bd2749cb87..9d31247e25 100644 --- a/packages/backend/src/core/AppLockService.ts +++ b/packages/backend/src/core/AppLockService.ts @@ -5,7 +5,7 @@ import { promisify } from 'node:util'; import { Inject, Injectable } from '@nestjs/common'; -import redisLock from 'redis-lock'; +import redisLock, { Unlock, NodeRedis } from 'redis-lock'; import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; @@ -17,13 +17,13 @@ const retryDelay = 100; @Injectable() export class AppLockService { - private lock: (key: string, timeout?: number, _?: (() => Promise) | undefined) => Promise<() => void>; + private lock: (key: string, timeout?: number) => Promise; constructor( @Inject(DI.redis) private redisClient: Redis.Redis, ) { - this.lock = promisify(redisLock(this.redisClient, retryDelay)); + this.lock = redisLock(adaptRedis(this.redisClient), retryDelay); } /** @@ -33,12 +33,36 @@ export class AppLockService { * @returns Unlock function */ @bindThis - public getApLock(uri: string, timeout = 30 * 1000): Promise<() => void> { + public getApLock(uri: string, timeout = 30 * 1000): Promise { return this.lock(`ap-object:${uri}`, timeout); } @bindThis - public getChartInsertLock(lockKey: string, timeout = 30 * 1000): Promise<() => void> { + public getChartInsertLock(lockKey: string, timeout = 30 * 1000): Promise { return this.lock(`chart-insert:${lockKey}`, timeout); } } + +/** + * Adapts an ioredis instance into something close enough to NodeRedis that it works with redis-lock. + */ +function adaptRedis(ioredis: Redis.Redis): NodeRedis { + return { + v4: true, + async set(key: string, value: string | number, opts?: { PX?: number, NX?: boolean }) { + if (opts) { + if (opts.PX != null && opts.NX) { + return ioredis.set(key, value, 'PX', opts.PX, 'NX'); + } else if (opts.PX != null) { + return ioredis.set(key, value, 'PX', opts.PX); + } else if (opts.NX) { + return ioredis.set(key, value, 'NX'); + } + } + return ioredis.set(key, value); + }, + async del(key: string) { + return await ioredis.del(key); + }, + }; +} diff --git a/packages/backend/src/core/AvatarDecorationService.ts b/packages/backend/src/core/AvatarDecorationService.ts index 4efd6122b1..aedb1d6a80 100644 --- a/packages/backend/src/core/AvatarDecorationService.ts +++ b/packages/backend/src/core/AvatarDecorationService.ts @@ -13,10 +13,13 @@ import { bindThis } from '@/decorators.js'; import { MemorySingleCache } from '@/misc/cache.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; +import { CacheManagementService, type ManagedMemorySingleCache } from '@/global/CacheManagementService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class AvatarDecorationService implements OnApplicationShutdown { - public cache: MemorySingleCache; + public cache: ManagedMemorySingleCache; constructor( @Inject(DI.redisForSub) @@ -28,29 +31,21 @@ export class AvatarDecorationService implements OnApplicationShutdown { private idService: IdService, private moderationLogService: ModerationLogService, private globalEventService: GlobalEventService, - ) { - this.cache = new MemorySingleCache(1000 * 60 * 30); // 30s + private readonly internalEventService: InternalEventService, + private readonly timeService: TimeService, - this.redisForSub.on('message', this.onMessage); + cacheManagementService: CacheManagementService, + ) { + this.cache = cacheManagementService.createMemorySingleCache('avatarDecorations', 1000 * 60 * 30); // 30s + + this.internalEventService.on('avatarDecorationCreated', this.onAvatarEvent); + this.internalEventService.on('avatarDecorationUpdated', this.onAvatarEvent); + this.internalEventService.on('avatarDecorationDeleted', this.onAvatarEvent); } @bindThis - private async onMessage(_: string, data: string): Promise { - const obj = JSON.parse(data); - - if (obj.channel === 'internal') { - const { type, body } = obj.message as GlobalEvents['internal']['payload']; - switch (type) { - case 'avatarDecorationCreated': - case 'avatarDecorationUpdated': - case 'avatarDecorationDeleted': { - this.cache.delete(); - break; - } - default: - break; - } - } + private onAvatarEvent(): void { + this.cache.delete(); } @bindThis @@ -60,7 +55,7 @@ export class AvatarDecorationService implements OnApplicationShutdown { ...options, }); - this.globalEventService.publishInternalEvent('avatarDecorationCreated', created); + await this.internalEventService.emit('avatarDecorationCreated', created); if (moderator) { this.moderationLogService.log(moderator, 'createAvatarDecoration', { @@ -76,14 +71,14 @@ export class AvatarDecorationService implements OnApplicationShutdown { public async update(id: MiAvatarDecoration['id'], params: Partial, moderator?: MiUser): Promise { const avatarDecoration = await this.avatarDecorationsRepository.findOneByOrFail({ id }); - const date = new Date(); + const date = this.timeService.date; await this.avatarDecorationsRepository.update(avatarDecoration.id, { updatedAt: date, ...params, }); const updated = await this.avatarDecorationsRepository.findOneByOrFail({ id: avatarDecoration.id }); - this.globalEventService.publishInternalEvent('avatarDecorationUpdated', updated); + await this.internalEventService.emit('avatarDecorationUpdated', updated); if (moderator) { this.moderationLogService.log(moderator, 'updateAvatarDecoration', { @@ -99,7 +94,7 @@ export class AvatarDecorationService implements OnApplicationShutdown { const avatarDecoration = await this.avatarDecorationsRepository.findOneByOrFail({ id }); await this.avatarDecorationsRepository.delete({ id: avatarDecoration.id }); - this.globalEventService.publishInternalEvent('avatarDecorationDeleted', avatarDecoration); + await this.internalEventService.emit('avatarDecorationDeleted', avatarDecoration); if (moderator) { this.moderationLogService.log(moderator, 'deleteAvatarDecoration', { @@ -119,7 +114,9 @@ export class AvatarDecorationService implements OnApplicationShutdown { @bindThis public dispose(): void { - this.redisForSub.off('message', this.onMessage); + this.internalEventService.off('avatarDecorationCreated', this.onAvatarEvent); + this.internalEventService.off('avatarDecorationUpdated', this.onAvatarEvent); + this.internalEventService.off('avatarDecorationDeleted', this.onAvatarEvent); } @bindThis diff --git a/packages/backend/src/core/BunnyService.ts b/packages/backend/src/core/BunnyService.ts index c9f8a427f5..7348eb016f 100644 --- a/packages/backend/src/core/BunnyService.ts +++ b/packages/backend/src/core/BunnyService.ts @@ -7,54 +7,58 @@ import * as https from 'node:https'; import * as fs from 'node:fs'; import { Readable } from 'node:stream'; import { finished } from 'node:stream/promises'; -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import type { MiMeta } from '@/models/Meta.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import Logger from '@/logger.js'; +import { DI } from '@/di-symbols.js'; @Injectable() export class BunnyService { - private bunnyCdnLogger: Logger; + private readonly bunnyCdnLogger: Logger; constructor( + @Inject(DI.meta) + private readonly meta: MiMeta, + private httpRequestService: HttpRequestService, ) { this.bunnyCdnLogger = new Logger('bunnycdn', 'blue'); } @bindThis - public getBunnyInfo(meta: MiMeta) { - if (!meta.objectStorageEndpoint || !meta.objectStorageBucket || !meta.objectStorageSecretKey) { + public getBunnyInfo() { + if (!this.meta.objectStorageEndpoint || !this.meta.objectStorageBucket || !this.meta.objectStorageSecretKey) { throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf90', 'Failed to use BunnyCDN, One of the required fields is missing.'); } return { - endpoint: meta.objectStorageEndpoint, + endpoint: this.meta.objectStorageEndpoint, /* The way S3 works is that the Secret Key is essentially the password for the API but Bunny calls their password AccessKey so we call it accessKey here. Bunny also doesn't specify a username/s3 access key when doing HTTP API requests so we end up not using our Access Key field from the form. */ - accessKey: meta.objectStorageSecretKey, - zone: meta.objectStorageBucket, - fullUrl: `https://${meta.objectStorageEndpoint}/${meta.objectStorageBucket}`, + accessKey: this.meta.objectStorageSecretKey, + zone: this.meta.objectStorageBucket, + fullUrl: `https://${this.meta.objectStorageEndpoint}/${this.meta.objectStorageBucket}`, }; } @bindThis - public usingBunnyCDN(meta: MiMeta) { - return meta.objectStorageEndpoint && meta.objectStorageEndpoint.endsWith('bunnycdn.com'); + public usingBunnyCDN() { + return this.meta.objectStorageEndpoint && this.meta.objectStorageEndpoint.endsWith('bunnycdn.com'); } @bindThis - public async upload(meta: MiMeta, path: string, input: fs.ReadStream | Buffer) { - const client = this.getBunnyInfo(meta); + public async upload(path: string, input: fs.ReadStream | Buffer) { + const client = this.getBunnyInfo(); // Required to convert the buffer from webpublic and thumbnail to a ReadableStream for PUT const data = Buffer.isBuffer(input) ? Readable.from(input) : input; - const agent = this.httpRequestService.getAgentByUrl(new URL(`${client.fullUrl}/${path}`), !meta.objectStorageUseProxy, true); + const agent = this.httpRequestService.getAgentByUrl(new URL(`${client.fullUrl}/${path}`), !this.meta.objectStorageUseProxy, true); // Seperation of path and host/domain is required here const options = { @@ -94,8 +98,8 @@ export class BunnyService { } @bindThis - public delete(meta: MiMeta, file: string) { - const client = this.getBunnyInfo(meta); + public delete(file: string) { + const client = this.getBunnyInfo(); return this.httpRequestService.send(`${client.fullUrl}/${file}`, { method: 'DELETE', headers: { AccessKey: client.accessKey } }); } } diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index d2fe988c6f..e58a012b6a 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -1,20 +1,39 @@ /* - * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors; originally based on code by syuilo and misskey-project * SPDX-License-Identifier: AGPL-3.0-only */ import { Inject, Injectable } from '@nestjs/common'; -import * as Redis from 'ioredis'; -import { In, IsNull } from 'typeorm'; -import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiNote, MiFollowing, NoteThreadMutingsRepository } from '@/models/_.js'; -import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; -import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; +import { In, IsNull, Brackets, MoreThan } from 'typeorm'; +import type { + BlockingsRepository, + FollowingsRepository, + MutingsRepository, + RenoteMutingsRepository, + MiUserProfile, + UserProfilesRepository, + UsersRepository, + MiFollowing, + NoteThreadMutingsRepository, + ChannelFollowingsRepository, + UserListMembershipsRepository, + UserListFavoritesRepository, +} from '@/models/_.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; +import type { MiUserListMembership } from '@/models/UserListMembership.js'; +import { isLocalUser, isRemoteUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import type { InternalEventTypes } from '@/core/GlobalEventService.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import * as Acct from '@/misc/acct.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { TimeService } from '@/global/TimeService.js'; +import { + CacheManagementService, + type ManagedMemoryKVCache, + type ManagedQuantumKVCache, +} from '@/global/CacheManagementService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; export interface FollowStats { @@ -24,210 +43,506 @@ export interface FollowStats { remoteFollowers: number; } -export interface CachedTranslation { - sourceLang: string | undefined; - text: string | undefined; -} - -export interface CachedTranslationEntity { - l?: string; - t?: string; - u?: number; -} - @Injectable() export class CacheService implements OnApplicationShutdown { - public userByIdCache: MemoryKVCache; - public localUserByNativeTokenCache: MemoryKVCache; - public localUserByIdCache: MemoryKVCache; - public uriPersonCache: MemoryKVCache; - public userProfileCache: QuantumKVCache; - public userMutingsCache: QuantumKVCache>; - public userBlockingCache: QuantumKVCache>; - public userBlockedCache: QuantumKVCache>; // NOTE: 「被」Blockキャッシュ - public renoteMutingsCache: QuantumKVCache>; - public threadMutingsCache: QuantumKVCache>; - public noteMutingsCache: QuantumKVCache>; - public userFollowingsCache: QuantumKVCache>>; - public userFollowersCache: QuantumKVCache>>; - public hibernatedUserCache: QuantumKVCache; - protected userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes - protected translationsCache: RedisKVCache; + /** + * Maps user IDs (key) to MiUser instances (value). + * This is the ONLY source for cached MiUser entities! + */ + public readonly userByIdCache: ManagedQuantumKVCache; + + /** + * Maps native tokens (key) to user IDs (value). + */ + public readonly nativeTokenCache: ManagedQuantumKVCache; + + /** + * Maps acct handles (key) to user IDs (value). + */ + public readonly userByAcctCache: ManagedQuantumKVCache; + + /** + * Maps user IDs (key) to MiUserProfile instances (value). + * This is the ONLY source for cached MiUserProfile entities! + */ + public readonly userProfileCache: ManagedQuantumKVCache; + + /** + * Maps user IDs (key) to the set of user IDs (value) muted by that user. + */ + public readonly userMutingsCache: ManagedQuantumKVCache>; + + /** + * Maps user IDs (key) to the set of user IDs (value) muting that user. + */ + public readonly userMutedCache: ManagedQuantumKVCache>; + + /** + * Maps user IDs (key) to the set of user IDs (value) blocked by that user. + */ + public readonly userBlockingCache: ManagedQuantumKVCache>; + + /** + * Maps user IDs (key) to the set of user IDs (value) blocking that user. + */ + public readonly userBlockedCache: ManagedQuantumKVCache>; + + /** + * Maps user IDs (key) to the map of list ID / MiUserListMembership instances (value) for all lists containing this user. + */ + public readonly userListMembershipsCache: ManagedQuantumKVCache>; + + /** + * Maps list IDs (key) to the map of user ID / MiUserListMembership instances (value) for all users in this list. + */ + public readonly listUserMembershipsCache: ManagedQuantumKVCache>; + + /** + * Maps user IDs (key) to the set of list IDs (value) that are favorited by that user + */ + public readonly userListFavoritesCache: ManagedQuantumKVCache>; + + /** + * Maps list IDs (key) to the set of user IDs (value) who have favorited this list. + */ + public readonly listUserFavoritesCache: ManagedQuantumKVCache>; + + /** + * Maps user IDs (key) to the set of user IDs (value) who's renotes are muted by that user. + */ + public readonly renoteMutingsCache: ManagedQuantumKVCache>; + + /** + * Maps user IDs (key) to the set of thread IDs (value) muted by that user. + */ + public readonly threadMutingsCache: ManagedQuantumKVCache>; + + /** + * Maps user IDs (key) to the set of note IDs (value) muted by that user. + */ + public readonly noteMutingsCache: ManagedQuantumKVCache>; + + /** + * Maps user IDs (key) to the map of user ID / MiFollowing instances (value) followed by that user. + */ + public readonly userFollowingsCache: ManagedQuantumKVCache>>; + + /** + * Maps user IDs (key) to the map of user ID / MiFollowing instances (value) following that user. + */ + public readonly userFollowersCache: ManagedQuantumKVCache>>; + + /** + * Maps user IDs (key) to hibernation state (value). + */ + public readonly hibernatedUserCache: ManagedQuantumKVCache; + + /** + * Maps user IDs (key) to follow statistics (value). + */ + public readonly userFollowStatsCache: ManagedMemoryKVCache; + + /** + * Maps user IDs (key) to the set of cahnnel IDs (value) followed by that user. + */ + public readonly userFollowingChannelsCache: ManagedQuantumKVCache>; constructor( - @Inject(DI.redis) - private redisClient: Redis.Redis, - - @Inject(DI.redisForSub) - private redisForSub: Redis.Redis, - @Inject(DI.usersRepository) - private usersRepository: UsersRepository, + private readonly usersRepository: UsersRepository, @Inject(DI.userProfilesRepository) - private userProfilesRepository: UserProfilesRepository, + private readonly userProfilesRepository: UserProfilesRepository, @Inject(DI.mutingsRepository) - private mutingsRepository: MutingsRepository, + private readonly mutingsRepository: MutingsRepository, @Inject(DI.blockingsRepository) - private blockingsRepository: BlockingsRepository, + private readonly blockingsRepository: BlockingsRepository, @Inject(DI.renoteMutingsRepository) - private renoteMutingsRepository: RenoteMutingsRepository, + private readonly renoteMutingsRepository: RenoteMutingsRepository, @Inject(DI.followingsRepository) - private followingsRepository: FollowingsRepository, + private readonly followingsRepository: FollowingsRepository, @Inject(DI.noteThreadMutingsRepository) private readonly noteThreadMutingsRepository: NoteThreadMutingsRepository, - private userEntityService: UserEntityService, + @Inject(DI.channelFollowingsRepository) + private readonly channelFollowingsRepository: ChannelFollowingsRepository, + + @Inject(DI.userListMembershipsRepository) + private readonly userListMembershipsRepository: UserListMembershipsRepository, + + @Inject(DI.userListFavoritesRepository) + private readonly userListFavoritesRepository: UserListFavoritesRepository, + private readonly internalEventService: InternalEventService, + private readonly cacheManagementService: CacheManagementService, + private readonly timeService: TimeService, ) { - //this.onMessage = this.onMessage.bind(this); - - this.userByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m - this.localUserByNativeTokenCache = new MemoryKVCache(1000 * 60 * 5); // 5m - this.localUserByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m - this.uriPersonCache = new MemoryKVCache(1000 * 60 * 5); // 5m - - this.userProfileCache = new QuantumKVCache(this.internalEventService, 'userProfile', { - lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.userProfilesRepository.findOneByOrFail({ userId: key }), - bulkFetcher: userIds => this.userProfilesRepository.findBy({ userId: In(userIds) }).then(ps => ps.map(p => [p.userId, p])), + this.userByIdCache = this.cacheManagementService.createQuantumKVCache('userById', { + lifetime: 1000 * 60 * 5, // 5m + fetcher: async (userId) => await this.usersRepository.findOneByOrFail({ id: userId }), + optionalFetcher: async (userId) => await this.usersRepository.findOneBy({ id: userId }), + bulkFetcher: async (userIds) => { + const users = await this.usersRepository.findBy({ id: In(userIds) }); + return users.map(user => [user.id, user]); + }, }); - this.userMutingsCache = new QuantumKVCache>(this.internalEventService, 'userMutings', { - lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.mutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))), - bulkFetcher: muterIds => this.mutingsRepository - .createQueryBuilder('muting') - .select('"muting"."muterId"', 'muterId') - .addSelect('array_agg("muting"."muteeId")', 'muteeIds') - .where({ muterId: In(muterIds) }) - .groupBy('muting.muterId') - .getRawMany<{ muterId: string, muteeIds: string[] }>() - .then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])), + this.nativeTokenCache = this.cacheManagementService.createQuantumKVCache('localUserByNativeToken', { + lifetime: 1000 * 60 * 5, // 5m + fetcher: async (token) => { + const { id } = await this.usersRepository + .createQueryBuilder('user') + .select('user.id') + .where({ token }) + .getOneOrFail() as { id: string }; + return id; + }, + optionalFetcher: async (token) => { + const result = await this.usersRepository + .createQueryBuilder('user') + .select('user.id') + .where({ token }) + .getOne() as { id: string } | null; + return result?.id; + }, + bulkFetcher: async (tokens) => { + const users = await this.usersRepository + .createQueryBuilder('user') + .select('user.id') + .addSelect('user.token') + .where({ token: In(tokens) }) + .getMany() as { id: string, token: string }[]; + return users.map(user => [user.token, user.id]); + }, }); - this.userBlockingCache = new QuantumKVCache>(this.internalEventService, 'userBlocking', { + this.userByAcctCache = this.cacheManagementService.createQuantumKVCache('userByAcct', { lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.blockingsRepository.find({ where: { blockerId: key }, select: ['blockeeId'] }).then(xs => new Set(xs.map(x => x.blockeeId))), - bulkFetcher: blockerIds => this.blockingsRepository - .createQueryBuilder('blocking') - .select('"blocking"."blockerId"', 'blockerId') - .addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds') - .where({ blockerId: In(blockerIds) }) - .groupBy('blocking.blockerId') - .getRawMany<{ blockerId: string, blockeeIds: string[] }>() - .then(ms => ms.map(m => [m.blockerId, new Set(m.blockeeIds)])), + fetcher: async (acct) => { + const parsed = Acct.parse(acct); + const { id } = await this.usersRepository + .createQueryBuilder('user') + .select('user.id') + .where({ + usernameLower: parsed.username.toLowerCase(), + host: parsed.host ?? IsNull(), + }) + .getOneOrFail(); + return id; + }, + optionalFetcher: async (acct) => { + const parsed = Acct.parse(acct); + const res = await this.usersRepository + .createQueryBuilder('user') + .select('user.id') + .where({ + usernameLower: parsed.username.toLowerCase(), + host: parsed.host ?? IsNull(), + }) + .getOne(); + return res?.id; + }, + // no bulkFetcher possible }); - this.userBlockedCache = new QuantumKVCache>(this.internalEventService, 'userBlocked', { + this.userProfileCache = this.cacheManagementService.createQuantumKVCache('userProfile', { lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.blockingsRepository.find({ where: { blockeeId: key }, select: ['blockerId'] }).then(xs => new Set(xs.map(x => x.blockerId))), - bulkFetcher: blockeeIds => this.blockingsRepository - .createQueryBuilder('blocking') - .select('"blocking"."blockeeId"', 'blockeeId') - .addSelect('array_agg("blocking"."blockerId")', 'blockerIds') - .where({ blockeeId: In(blockeeIds) }) - .groupBy('blocking.blockeeId') - .getRawMany<{ blockeeId: string, blockerIds: string[] }>() - .then(ms => ms.map(m => [m.blockeeId, new Set(m.blockerIds)])), + fetcher: async userId => await this.userProfilesRepository.findOneByOrFail({ userId }), + optionalFetcher: async userId => await this.userProfilesRepository.findOneBy({ userId }), + bulkFetcher: async userIds => { + const profiles = await this.userProfilesRepository.findBy({ userId: In(userIds) }); + return profiles.map(profile => [profile.userId, profile]); + }, }); - this.renoteMutingsCache = new QuantumKVCache>(this.internalEventService, 'renoteMutings', { - lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.renoteMutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))), - bulkFetcher: muterIds => this.renoteMutingsRepository - .createQueryBuilder('muting') - .select('"muting"."muterId"', 'muterId') - .addSelect('array_agg("muting"."muteeId")', 'muteeIds') - .where({ muterId: In(muterIds) }) - .groupBy('muting.muterId') - .getRawMany<{ muterId: string, muteeIds: string[] }>() - .then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])), + this.userMutingsCache = this.cacheManagementService.createQuantumKVCache>('userMutings', { + lifetime: 1000 * 60 * 30, // 3m (workaround for mute expiration) + fetcher: async muterId => { + const mutings = await this.mutingsRepository.find({ where: { muterId: muterId }, select: ['muteeId'] }); + return new Set(mutings.map(muting => muting.muteeId)); + }, + // no optionalFetcher needed + bulkFetcher: async muterIds => { + const mutings = await this.mutingsRepository + .createQueryBuilder('muting') + .select('"muting"."muterId"', 'muterId') + .addSelect('array_agg("muting"."muteeId")', 'muteeIds') + .where({ muterId: In(muterIds) }) + .andWhere(new Brackets(qb => qb + .orWhere({ expiresAt: IsNull() }) + .orWhere({ expiresAt: MoreThan(this.timeService.date) }))) + .groupBy('muting.muterId') + .getRawMany<{ muterId: string, muteeIds: string[] }>(); + return mutings.map(muting => [muting.muterId, new Set(muting.muteeIds)]); + }, }); - this.threadMutingsCache = new QuantumKVCache>(this.internalEventService, 'threadMutings', { - lifetime: 1000 * 60 * 30, // 30m - fetcher: muterId => this.noteThreadMutingsRepository - .find({ where: { userId: muterId, isPostMute: false }, select: { threadId: true } }) - .then(ms => new Set(ms.map(m => m.threadId))), - bulkFetcher: muterIds => this.noteThreadMutingsRepository - .createQueryBuilder('muting') - .select('"muting"."userId"', 'userId') - .addSelect('array_agg("muting"."threadId")', 'threadIds') - .groupBy('"muting"."userId"') - .where({ userId: In(muterIds), isPostMute: false }) - .getRawMany<{ userId: string, threadIds: string[] }>() - .then(ms => ms.map(m => [m.userId, new Set(m.threadIds)])), + this.userMutedCache = this.cacheManagementService.createQuantumKVCache>('userMuted', { + lifetime: 1000 * 60 * 30, // 3m (workaround for mute expiration) + fetcher: async muteeId => { + const mutings = await this.mutingsRepository.find({ where: { muteeId }, select: ['muterId'] }); + return new Set(mutings.map(muting => muting.muterId)); + }, + // no optionalFetcher needed + bulkFetcher: async muteeIds => { + const mutings = await this.mutingsRepository + .createQueryBuilder('muting') + .select('"muting"."muteeId"', 'muteeId') + .addSelect('array_agg("muting"."muterId")', 'muterIds') + .where({ muteeId: In(muteeIds) }) + .andWhere(new Brackets(qb => qb + .orWhere({ expiresAt: IsNull() }) + .orWhere({ expiresAt: MoreThan(this.timeService.date) }))) + .groupBy('muting.muteeId') + .getRawMany<{ muteeId: string, muterIds: string[] }>(); + return mutings.map(muting => [muting.muteeId, new Set(muting.muterIds)]); + }, }); - this.noteMutingsCache = new QuantumKVCache>(this.internalEventService, 'noteMutings', { + this.userBlockingCache = this.cacheManagementService.createQuantumKVCache>('userBlocking', { lifetime: 1000 * 60 * 30, // 30m - fetcher: muterId => this.noteThreadMutingsRepository - .find({ where: { userId: muterId, isPostMute: true }, select: { threadId: true } }) - .then(ms => new Set(ms.map(m => m.threadId))), - bulkFetcher: muterIds => this.noteThreadMutingsRepository - .createQueryBuilder('muting') - .select('"muting"."userId"', 'userId') - .addSelect('array_agg("muting"."threadId")', 'threadIds') - .groupBy('"muting"."userId"') - .where({ userId: In(muterIds), isPostMute: true }) - .getRawMany<{ userId: string, threadIds: string[] }>() - .then(ms => ms.map(m => [m.userId, new Set(m.threadIds)])), + fetcher: async blockerId => { + const blockings = await this.blockingsRepository.find({ where: { blockerId }, select: ['blockeeId'] }); + return new Set(blockings.map(blocking => blocking.blockeeId)); + }, + // no optionalFetcher needed + bulkFetcher: async blockerIds => { + const blockings = await this.blockingsRepository + .createQueryBuilder('blocking') + .select('"blocking"."blockerId"', 'blockerId') + .addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds') + .where({ blockerId: In(blockerIds) }) + .groupBy('blocking.blockerId') + .getRawMany<{ blockerId: string, blockeeIds: string[] }>(); + return blockings.map(blocking => [blocking.blockerId, new Set(blocking.blockeeIds)]); + }, }); - this.userFollowingsCache = new QuantumKVCache>>(this.internalEventService, 'userFollowings', { + this.userBlockedCache = this.cacheManagementService.createQuantumKVCache>('userBlocked', { lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.followingsRepository.findBy({ followerId: key }).then(xs => new Map(xs.map(f => [f.followeeId, f]))), - bulkFetcher: followerIds => this.followingsRepository - .findBy({ followerId: In(followerIds) }) - .then(fs => fs - .reduce((groups, f) => { - let group = groups.get(f.followerId); - if (!group) { - group = new Map(); - groups.set(f.followerId, group); - } - group.set(f.followeeId, f); - return groups; - }, new Map>>)), + fetcher: async blockeeId => { + const blockings = await this.blockingsRepository.find({ where: { blockeeId: blockeeId }, select: ['blockerId'] }); + return new Set(blockings.map(blocking => blocking.blockerId)); + }, + // no optionalFetcher needed + bulkFetcher: async blockeeIds => { + const blockings = await this.blockingsRepository + .createQueryBuilder('blocking') + .select('"blocking"."blockeeId"', 'blockeeId') + .addSelect('array_agg("blocking"."blockerId")', 'blockerIds') + .where({ blockeeId: In(blockeeIds) }) + .groupBy('blocking.blockeeId') + .getRawMany<{ blockeeId: string, blockerIds: string[] }>(); + return blockings.map(blocking => [blocking.blockeeId, new Set(blocking.blockerIds)]); + }, }); - this.userFollowersCache = new QuantumKVCache>>(this.internalEventService, 'userFollowers', { - lifetime: 1000 * 60 * 30, // 30m - fetcher: followeeId => this.followingsRepository.findBy({ followeeId: followeeId }).then(xs => new Map(xs.map(x => [x.followerId, x]))), - bulkFetcher: followeeIds => this.followingsRepository - .findBy({ followeeId: In(followeeIds) }) - .then(fs => fs - .reduce((groups, f) => { - let group = groups.get(f.followeeId); - if (!group) { - group = new Map(); - groups.set(f.followeeId, group); - } - group.set(f.followerId, f); - return groups; - }, new Map>>)), - }); - - this.hibernatedUserCache = new QuantumKVCache(this.internalEventService, 'hibernatedUsers', { + this.userListMembershipsCache = this.cacheManagementService.createQuantumKVCache>('userListMemberships', { lifetime: 1000 * 60 * 30, // 30m fetcher: async userId => { - const { isHibernated } = await this.usersRepository.findOneOrFail({ - where: { id: userId }, - select: { isHibernated: true }, - }); + const memberships = await this.userListMembershipsRepository.findBy({ userId }); + return new Map(memberships.map(membership => [membership.userListId, membership])); + }, + // no optionalFetcher needed + bulkFetcher: async userIds => { + const groups = new Map>; + + const memberships = await this.userListMembershipsRepository.findBy({ userId: In(userIds) }); + for (const membership of memberships) { + let listsForUser = groups.get(membership.userId); + if (!listsForUser) { + listsForUser = new Map(); + groups.set(membership.userId, listsForUser); + } + listsForUser.set(membership.userListId, membership); + } + + return groups; + }, + }); + + this.listUserMembershipsCache = this.cacheManagementService.createQuantumKVCache>('listUserMemberships', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async userListId => { + const memberships = await this.userListMembershipsRepository.findBy({ userListId }); + return new Map(memberships.map(membership => [membership.userId, membership])); + }, + // no optionalFetcher needed + bulkFetcher: async userListIds => { + const memberships = await this.userListMembershipsRepository.findBy({ userListId: In(userListIds) }); + const groups = new Map>(); + for (const membership of memberships) { + let usersForList = groups.get(membership.userListId); + if (!usersForList) { + usersForList = new Map(); + groups.set(membership.userListId, usersForList); + } + usersForList.set(membership.userId, membership); + } + return groups; + }, + }); + + this.userListFavoritesCache = cacheManagementService.createQuantumKVCache>('userListFavorites', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async userId => { + const favorites = await this.userListFavoritesRepository.find({ where: { userId }, select: ['userListId'] }); + return new Set(favorites.map(favorites => favorites.userListId)); + }, + // no optionalFetcher needed + bulkFetcher: async userIds => { + const favorites = await this.userListFavoritesRepository + .createQueryBuilder('favorite') + .select('"favorite"."userId"', 'userId') + .addSelect('array_agg("favorite"."userListId")', 'userListIds') + .where({ userId: In(userIds) }) + .groupBy('favorite.userId') + .getRawMany<{ userId: string, userListIds: string[] }>(); + return favorites.map(favorite => [favorite.userId, new Set(favorite.userListIds)]); + }, + }); + + this.listUserFavoritesCache = cacheManagementService.createQuantumKVCache>('listUserFavorites', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async userListId => { + const favorites = await this.userListFavoritesRepository.find({ where: { userListId }, select: ['userId'] }); + return new Set(favorites.map(favorite => favorite.userId)); + }, + // no optionalFetcher needed + bulkFetcher: async userListIds => { + const favorites = await this.userListFavoritesRepository + .createQueryBuilder('favorite') + .select('"favorite"."userListId"', 'userListId') + .addSelect('array_agg("favorite"."userId")', 'userIds') + .where({ userListId: In(userListIds) }) + .groupBy('favorite.userListId') + .getRawMany<{ userListId: string, userIds: string[] }>(); + return favorites.map(favorite => [favorite.userListId, new Set(favorite.userIds)]); + }, + }); + + this.renoteMutingsCache = this.cacheManagementService.createQuantumKVCache>('renoteMutings', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async muterId => { + const mutings = await this.renoteMutingsRepository.find({ where: { muterId: muterId }, select: ['muteeId'] }); + return new Set(mutings.map(muting => muting.muteeId)); + }, + // no optionalFetcher needed + bulkFetcher: async muterIds => { + const mutings = await this.renoteMutingsRepository + .createQueryBuilder('muting') + .select('"muting"."muterId"', 'muterId') + .addSelect('array_agg("muting"."muteeId")', 'muteeIds') + .where({ muterId: In(muterIds) }) + .groupBy('muting.muterId') + .getRawMany<{ muterId: string, muteeIds: string[] }>(); + return mutings.map(muting => [muting.muterId, new Set(muting.muteeIds)]); + }, + }); + + this.threadMutingsCache = this.cacheManagementService.createQuantumKVCache>('threadMutings', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async muterId => { + const mutings = await this.noteThreadMutingsRepository.find({ where: { userId: muterId, isPostMute: false }, select: { threadId: true } }); + return new Set(mutings.map(muting => muting.threadId)); + }, + // no optionalFetcher needed + bulkFetcher: async muterIds => { + const mutings = await this.noteThreadMutingsRepository + .createQueryBuilder('muting') + .select('"muting"."userId"', 'userId') + .addSelect('array_agg("muting"."threadId")', 'threadIds') + .groupBy('"muting"."userId"') + .where({ userId: In(muterIds), isPostMute: false }) + .getRawMany<{ userId: string, threadIds: string[] }>(); + return mutings.map(muting => [muting.userId, new Set(muting.threadIds)]); + }, + }); + + this.noteMutingsCache = this.cacheManagementService.createQuantumKVCache>('noteMutings', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async muterId => { + const mutings = await this.noteThreadMutingsRepository.find({ where: { userId: muterId, isPostMute: true }, select: { threadId: true } }); + return new Set(mutings.map(mutings => mutings.threadId)); + }, + // no optionalFetcher needed + bulkFetcher: async muterIds => { + const mutings = await this.noteThreadMutingsRepository + .createQueryBuilder('muting') + .select('"muting"."userId"', 'userId') + .addSelect('array_agg("muting"."threadId")', 'threadIds') + .groupBy('"muting"."userId"') + .where({ userId: In(muterIds), isPostMute: true }) + .getRawMany<{ userId: string, threadIds: string[] }>(); + return mutings.map(muting => [muting.userId, new Set(muting.threadIds)]); + }, + }); + + this.userFollowingsCache = this.cacheManagementService.createQuantumKVCache>>('userFollowings', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async followerId => { + const followings = await this.followingsRepository.findBy({ followerId: followerId }); + return new Map(followings.map(following => [following.followeeId, following])); + }, + // no optionalFetcher needed + bulkFetcher: async followerIds => { + const groups = new Map>>(); + + const followings = await this.followingsRepository.findBy({ followerId: In(followerIds) }); + for (const following of followings) { + let group = groups.get(following.followerId); + if (!group) { + group = new Map(); + groups.set(following.followerId, group); + } + group.set(following.followeeId, following); + } + + return groups; + }, + }); + + this.userFollowersCache = this.cacheManagementService.createQuantumKVCache>>('userFollowers', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async followeeId => { + const followings = await this.followingsRepository.findBy({ followeeId: followeeId }); + return new Map(followings.map(following => [following.followerId, following])); + }, + // no optionalFetcher needed + bulkFetcher: async followeeIds => { + const groups = new Map>>(); + + const followings = await this.followingsRepository.findBy({ followeeId: In(followeeIds) }); + for (const following of followings) { + let group = groups.get(following.followeeId); + if (!group) { + group = new Map(); + groups.set(following.followeeId, group); + } + group.set(following.followerId, following); + } + + return groups; + }, + }); + + this.hibernatedUserCache = this.cacheManagementService.createQuantumKVCache('hibernatedUsers', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async userId => { + const { isHibernated } = await this.usersRepository.findOneOrFail({ where: { id: userId }, select: { isHibernated: true } }); return isHibernated; }, + optionalFetcher: async userId => { + const result = await this.usersRepository.findOne({ where: { id: userId }, select: { isHibernated: true } }); + return result?.isHibernated; + }, bulkFetcher: async userIds => { - const results = await this.usersRepository.find({ - where: { id: In(userIds) }, - select: { id: true, isHibernated: true }, - }); + const results = await this.usersRepository.find({ where: { id: In(userIds) }, select: { id: true, isHibernated: true } }); return results.map(({ id, isHibernated }) => [id, isHibernated]); }, onChanged: async userIds => { @@ -238,10 +553,7 @@ export class CacheService implements OnApplicationShutdown { for (const uid of userIds) { const toAdd: MiUser[] = []; - const localUserById = this.localUserByIdCache.get(uid); - if (localUserById) toAdd.push(localUserById); - - const userById = this.userByIdCache.get(uid); + const userById = this.userByIdCache.getMaybe(uid); if (userById) toAdd.push(userById); if (toAdd.length > 0) { @@ -257,8 +569,8 @@ export class CacheService implements OnApplicationShutdown { for (const { id, isHibernated } of hibernations) { const users = userObjects.get(id); if (users) { - for (const u of users) { - u.isHibernated = isHibernated; + for (const user of users) { + user.isHibernated = isHibernated; } } } @@ -266,89 +578,101 @@ export class CacheService implements OnApplicationShutdown { }, }); - this.translationsCache = new RedisKVCache(this.redisClient, 'translations', { - lifetime: 1000 * 60 * 60 * 24 * 7, // 1 week, - memoryCacheLifetime: 1000 * 60, // 1 minute + this.userFollowStatsCache = this.cacheManagementService.createMemoryKVCache('followStats', 1000 * 60 * 10); // 10 minutes + + this.userFollowingChannelsCache = this.cacheManagementService.createQuantumKVCache>('userFollowingChannels', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async (followerId) => { + const followings = await this.channelFollowingsRepository.find({ where: { followerId: followerId }, select: ['followeeId'] }); + return new Set(followings.map(following => following.followeeId)); + }, + // no optionalFetcher needed + bulkFetcher: async followerIds => { + const followings = await this.channelFollowingsRepository + .createQueryBuilder('following') + .select('"following"."followerId"', 'followerId') + .addSelect('array_agg("following"."followeeId")', 'followeeIds') + .where({ followerId: In(followerIds) }) + .groupBy('following.followerId') + .getRawMany<{ followerId: string, followeeIds: string[] }>(); + return followings.map(following => [following.followerId, new Set(following.followeeIds)]); + }, }); - // NOTE: チャンネルのフォロー状況キャッシュはChannelFollowingServiceで行っている - + this.internalEventService.on('usersUpdated', this.onUserEvent); this.internalEventService.on('userChangeSuspendedState', this.onUserEvent); this.internalEventService.on('userChangeDeletedState', this.onUserEvent); this.internalEventService.on('remoteUserUpdated', this.onUserEvent); this.internalEventService.on('localUserUpdated', this.onUserEvent); - this.internalEventService.on('userChangeSuspendedState', this.onUserEvent); + this.internalEventService.on('userUpdated', this.onUserEvent); this.internalEventService.on('userTokenRegenerated', this.onTokenEvent); this.internalEventService.on('follow', this.onFollowEvent); this.internalEventService.on('unfollow', this.onFollowEvent); + // For these, only listen to local events because quantum cache handles the sync. + this.internalEventService.on('followChannel', this.onChannelEvent, { ignoreRemote: true }); + this.internalEventService.on('unfollowChannel', this.onChannelEvent, { ignoreRemote: true }); + this.internalEventService.on('updateUserProfile', this.onProfileEvent, { ignoreRemote: true }); + this.internalEventService.on('userListMemberAdded', this.onListMemberEvent, { ignoreRemote: true }); + this.internalEventService.on('userListMemberUpdated', this.onListMemberEvent, { ignoreRemote: true }); + this.internalEventService.on('userListMemberRemoved', this.onListMemberEvent, { ignoreRemote: true }); + this.internalEventService.on('userListMemberBulkAdded', this.onListMemberEvent, { ignoreRemote: true }); + this.internalEventService.on('userListMemberBulkUpdated', this.onListMemberEvent, { ignoreRemote: true }); + this.internalEventService.on('userListMemberBulkRemoved', this.onListMemberEvent, { ignoreRemote: true }); } @bindThis - private async onUserEvent(body: InternalEventTypes[E], _: E, isLocal: boolean): Promise { - { - { - { - const user = await this.usersRepository.findOneBy({ id: body.id }); - if (user == null) { - this.userByIdCache.delete(body.id); - this.localUserByIdCache.delete(body.id); - for (const [k, v] of this.uriPersonCache.entries) { - if (v.value?.id === body.id) { - this.uriPersonCache.delete(k); - } - } - if (isLocal) { - await Promise.all([ - this.userProfileCache.delete(body.id), - this.userMutingsCache.delete(body.id), - this.userBlockingCache.delete(body.id), - this.userBlockedCache.delete(body.id), - this.renoteMutingsCache.delete(body.id), - this.userFollowingsCache.delete(body.id), - this.userFollowersCache.delete(body.id), - this.hibernatedUserCache.delete(body.id), - this.threadMutingsCache.delete(body.id), - this.noteMutingsCache.delete(body.id), - ]); - } - } else { - this.userByIdCache.set(user.id, user); - for (const [k, v] of this.uriPersonCache.entries) { - if (v.value?.id === user.id) { - this.uriPersonCache.set(k, user); - } - } - if (this.userEntityService.isLocalUser(user)) { - this.localUserByNativeTokenCache.set(user.token!, user); - this.localUserByIdCache.set(user.id, user); - } - } - } - } - } + private async onUserEvent(body: InternalEventTypes[E], _: E, isLocal: boolean): Promise { + // Local instance is responsible for expanding these events into the appropriate Quantum events + if (!isLocal) return; + + const ids = 'ids' in body ? body.ids : [body.id]; + if (ids.length === 0) return; + + // Contains IDs of all lists where this user is a member. + const userListMemberships = this.listUserMembershipsCache + .entries() + .filter(e => ids.some(id => e[1].has(id))) + .map(e => e[0]) + .toArray(); + + await Promise.all([ + this.userByIdCache.deleteMany(ids), + this.userProfileCache.deleteMany(ids), + this.userMutingsCache.deleteMany(ids), + this.userMutedCache.deleteMany(ids), + this.userBlockingCache.deleteMany(ids), + this.userBlockedCache.deleteMany(ids), + this.renoteMutingsCache.deleteMany(ids), + this.userFollowingsCache.deleteMany(ids), + this.userFollowersCache.deleteMany(ids), + this.hibernatedUserCache.deleteMany(ids), + this.threadMutingsCache.deleteMany(ids), + this.noteMutingsCache.deleteMany(ids), + this.userListMembershipsCache.deleteMany(ids), + this.listUserMembershipsCache.deleteMany(userListMemberships), + ]); } @bindThis - private async onTokenEvent(body: InternalEventTypes[E]): Promise { - { - { - { - const user = await this.usersRepository.findOneByOrFail({ id: body.id }) as MiLocalUser; - this.localUserByNativeTokenCache.delete(body.oldToken); - this.localUserByNativeTokenCache.set(body.newToken, user); - } - } - } + private async onTokenEvent(body: InternalEventTypes[E], _: E, isLocal: boolean): Promise { + // Local instance is responsible for expanding these events into the appropriate Quantum events + if (!isLocal) return; + + await Promise.all([ + this.nativeTokenCache.delete(body.oldToken), + this.nativeTokenCache.set(body.newToken, body.id), + ]); } @bindThis private async onFollowEvent(body: InternalEventTypes[E], type: E): Promise { { + // TODO should we filter for local/remote events? switch (type) { case 'follow': { - const follower = this.userByIdCache.get(body.followerId); + const follower = this.userByIdCache.getMaybe(body.followerId); if (follower) follower.followingCount++; - const followee = this.userByIdCache.get(body.followeeId); + const followee = this.userByIdCache.getMaybe(body.followeeId); if (followee) followee.followersCount++; await Promise.all([ this.userFollowingsCache.delete(body.followerId), @@ -359,9 +683,9 @@ export class CacheService implements OnApplicationShutdown { break; } case 'unfollow': { - const follower = this.userByIdCache.get(body.followerId); + const follower = this.userByIdCache.getMaybe(body.followerId); if (follower) follower.followingCount--; - const followee = this.userByIdCache.get(body.followeeId); + const followee = this.userByIdCache.getMaybe(body.followeeId); if (followee) followee.followersCount--; await Promise.all([ this.userFollowingsCache.delete(body.followerId), @@ -376,31 +700,112 @@ export class CacheService implements OnApplicationShutdown { } @bindThis - public findUserById(userId: MiUser['id']) { - return this.userByIdCache.fetch(userId, () => this.usersRepository.findOneByOrFail({ id: userId })); + private async onChannelEvent(body: InternalEventTypes[E]): Promise { + await this.userFollowingChannelsCache.delete(body.userId); } @bindThis - public async findLocalUserById(userId: MiUser['id']): Promise { - return await this.localUserByIdCache.fetchMaybe(userId, async () => { - return await this.usersRepository.findOneBy({ id: userId, host: IsNull() }) as MiLocalUser | null ?? undefined; - }) ?? null; + private async onProfileEvent(body: InternalEventTypes[E]): Promise { + await this.userProfileCache.delete(body.userId); } @bindThis - public async findRemoteUserById(userId: MiUser['id']): Promise { + private async onListMemberEvent(body: InternalEventTypes[E]): Promise { + const userListIds = 'userListIds' in body ? body.userListIds : [body.userListId]; + await Promise.all([ + this.userListMembershipsCache.delete(body.memberId), + this.listUserMembershipsCache.deleteMany(userListIds), + ]); + } + + @bindThis + public async findUserById(userId: MiUser['id']): Promise { + return await this.userByIdCache.fetch(userId); + } + + @bindThis + public async findUsersById(userIds: Iterable): Promise> { + return new Map(await this.userByIdCache.fetchMany(userIds)); + } + + @bindThis + public async findOptionalUserById(userId: MiUser['id']): Promise { + return await this.userByIdCache.fetchMaybe(userId); + } + + @bindThis + public async findUserByAcct(acct: string | Acct.Acct): Promise { + acct = typeof(acct) === 'string' ? acct : Acct.toString(acct); + const id = await this.userByAcctCache.fetch(acct); + return await this.findUserById(id); + } + + @bindThis + public async findOptionalUserByAcct(acct: string | Acct.Acct): Promise { + acct = typeof(acct) === 'string' ? acct : Acct.toString(acct); + + const id = await this.userByAcctCache.fetchMaybe(acct); + if (id == null) return undefined; + + return await this.findOptionalUserById(id); + } + + @bindThis + public async findLocalUserById(userId: MiUser['id']): Promise { const user = await this.findUserById(userId); - if (user.host == null) { - return null; + if (!isLocalUser(user)) { + throw new IdentifiableError('aeac1339-2550-4521-a8e3-781f06d98656', 'User is not local'); } - return user as MiRemoteUser; + return user; } @bindThis - public findOptionalUserById(userId: MiUser['id']) { - return this.userByIdCache.fetchMaybe(userId, async () => await this.usersRepository.findOneBy({ id: userId }) ?? undefined); + public async findOptionalLocalUserById(userId: MiUser['id']): Promise { + const user = await this.findOptionalUserById(userId); + + if (user && !isLocalUser(user)) { + throw new IdentifiableError('aeac1339-2550-4521-a8e3-781f06d98656', 'User is not local'); + } + + return user; + } + + @bindThis + public async findLocalUserByNativeToken(token: string): Promise { + const id = await this.nativeTokenCache.fetch(token); + return await this.findLocalUserById(id); + } + + @bindThis + public async findOptionalLocalUserByNativeToken(token: string): Promise { + const id = await this.nativeTokenCache.fetchMaybe(token); + if (id == null) return undefined; + + return await this.findOptionalLocalUserById(id); + } + + @bindThis + public async findRemoteUserById(userId: MiUser['id']): Promise { + const user = await this.findUserById(userId); + + if (!isRemoteUser(user)) { + throw new IdentifiableError('aeac1339-2550-4521-a8e3-781f06d98656', 'User is not remote'); + } + + return user; + } + + @bindThis + public async findOptionalRemoteUserById(userId: MiUser['id']): Promise { + const user = await this.findOptionalUserById(userId); + + if (user && !isRemoteUser(user)) { + throw new IdentifiableError('aeac1339-2550-4521-a8e3-781f06d98656', 'User is not remote'); + } + + return user; } @bindThis @@ -449,69 +854,13 @@ export class CacheService implements OnApplicationShutdown { }); } - @bindThis - public async getCachedTranslation(note: MiNote, targetLang: string): Promise { - const cacheKey = `${note.id}@${targetLang}`; - - // Use cached translation, if present and up-to-date - const cached = await this.translationsCache.get(cacheKey); - if (cached && cached.u === note.updatedAt?.valueOf()) { - return { - sourceLang: cached.l, - text: cached.t, - }; - } - - // No cache entry :( - return null; - } - - @bindThis - public async setCachedTranslation(note: MiNote, targetLang: string, translation: CachedTranslation): Promise { - const cacheKey = `${note.id}@${targetLang}`; - - await this.translationsCache.set(cacheKey, { - l: translation.sourceLang, - t: translation.text, - u: note.updatedAt?.valueOf(), - }); - } - - @bindThis - public async getUsers(userIds: Iterable): Promise> { - const users = new Map; - - const toFetch: string[] = []; - for (const userId of userIds) { - const fromCache = this.userByIdCache.get(userId); - if (fromCache) { - users.set(userId, fromCache); - } else { - toFetch.push(userId); - } - } - - if (toFetch.length > 0) { - const fetched = await this.usersRepository.findBy({ - id: In(toFetch), - }); - - for (const user of fetched) { - users.set(user.id, user); - this.userByIdCache.set(user.id, user); - } - } - - return users; - } - @bindThis public async isFollowing(follower: string | { id: string }, followee: string | { id: string }): Promise { const followerId = typeof(follower) === 'string' ? follower : follower.id; const followeeId = typeof(followee) === 'string' ? followee : followee.id; // This lets us use whichever one is in memory, falling back to DB fetch via userFollowingsCache. - return this.userFollowersCache.get(followeeId)?.has(followerId) + return this.userFollowersCache.getMaybe(followeeId)?.has(followerId) ?? (await this.userFollowingsCache.fetch(followerId)).has(followeeId); } @@ -521,7 +870,7 @@ export class CacheService implements OnApplicationShutdown { @bindThis public async getHibernatedFollowers(followeeId: string): Promise { const followers = await this.getFollowersWithHibernation(followeeId); - return followers.filter(f => f.isFollowerHibernated); + return followers.filter(follower => follower.isFollowerHibernated); } /** @@ -530,7 +879,7 @@ export class CacheService implements OnApplicationShutdown { @bindThis public async getNonHibernatedFollowers(followeeId: string): Promise { const followers = await this.getFollowersWithHibernation(followeeId); - return followers.filter(f => !f.isFollowerHibernated); + return followers.filter(follower => !follower.isFollowerHibernated); } /** @@ -540,14 +889,14 @@ export class CacheService implements OnApplicationShutdown { @bindThis public async getFollowersWithHibernation(followeeId: string): Promise { const followers = await this.userFollowersCache.fetch(followeeId); - const hibernations = await this.hibernatedUserCache.fetchMany(followers.keys()).then(fs => fs.reduce((map, f) => { - map.set(f[0], f[1]); - return map; - }, new Map)); - return Array.from(followers.values()).map(following => ({ - ...following, - isFollowerHibernated: hibernations.get(following.followerId) ?? false, - })); + const hibernations = new Map(await this.hibernatedUserCache.fetchMany(followers.keys())); + return followers + .values() + .map(following => ({ + ...following, + isFollowerHibernated: hibernations.get(following.followerId) ?? false, + })) + .toArray(); } /** @@ -556,54 +905,39 @@ export class CacheService implements OnApplicationShutdown { @bindThis public async refreshFollowRelationsFor(userId: string): Promise { const followings = await this.userFollowingsCache.refresh(userId); - const followees = Array.from(followings.values()).map(f => f.followeeId); + const followees = followings.values().map(following => following.followeeId); await this.userFollowersCache.deleteMany(followees); } @bindThis public clear(): void { - this.userByIdCache.clear(); - this.localUserByNativeTokenCache.clear(); - this.localUserByIdCache.clear(); - this.uriPersonCache.clear(); - this.userProfileCache.clear(); - this.userMutingsCache.clear(); - this.userBlockingCache.clear(); - this.userBlockedCache.clear(); - this.renoteMutingsCache.clear(); - this.userFollowingsCache.clear(); - this.userFollowStatsCache.clear(); - this.translationsCache.clear(); + this.cacheManagementService.clear(); } @bindThis public dispose(): void { + this.internalEventService.off('usersUpdated', this.onUserEvent); this.internalEventService.off('userChangeSuspendedState', this.onUserEvent); this.internalEventService.off('userChangeDeletedState', this.onUserEvent); this.internalEventService.off('remoteUserUpdated', this.onUserEvent); this.internalEventService.off('localUserUpdated', this.onUserEvent); - this.internalEventService.off('userChangeSuspendedState', this.onUserEvent); + this.internalEventService.off('userUpdated', this.onUserEvent); this.internalEventService.off('userTokenRegenerated', this.onTokenEvent); this.internalEventService.off('follow', this.onFollowEvent); this.internalEventService.off('unfollow', this.onFollowEvent); - this.userByIdCache.dispose(); - this.localUserByNativeTokenCache.dispose(); - this.localUserByIdCache.dispose(); - this.uriPersonCache.dispose(); - this.userProfileCache.dispose(); - this.userMutingsCache.dispose(); - this.userBlockingCache.dispose(); - this.userBlockedCache.dispose(); - this.renoteMutingsCache.dispose(); - this.threadMutingsCache.dispose(); - this.noteMutingsCache.dispose(); - this.userFollowingsCache.dispose(); - this.userFollowersCache.dispose(); - this.hibernatedUserCache.dispose(); + this.internalEventService.off('followChannel', this.onChannelEvent); + this.internalEventService.off('unfollowChannel', this.onChannelEvent); + this.internalEventService.off('updateUserProfile', this.onProfileEvent); + this.internalEventService.off('userListMemberAdded', this.onListMemberEvent); + this.internalEventService.off('userListMemberUpdated', this.onListMemberEvent); + this.internalEventService.off('userListMemberRemoved', this.onListMemberEvent); + this.internalEventService.off('userListMemberBulkAdded', this.onListMemberEvent); + this.internalEventService.off('userListMemberBulkUpdated', this.onListMemberEvent); + this.internalEventService.off('userListMemberBulkRemoved', this.onListMemberEvent); } @bindThis - public onApplicationShutdown(signal?: string | undefined): void { + public onApplicationShutdown(): void { this.dispose(); } } diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index 020984a37f..a0ea909cf2 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -10,23 +10,13 @@ import { MetaService } from '@/core/MetaService.js'; import { MiMeta } from '@/models/Meta.js'; import Logger from '@/logger.js'; import { LoggerService } from '@/core/LoggerService.js'; -import { CaptchaError } from '@/misc/captcha-error.js'; +import { CaptchaError, captchaErrorCodes } from '@/misc/captcha-error.js'; export { CaptchaError } from '@/misc/captcha-error.js'; export const supportedCaptchaProviders = ['none', 'hcaptcha', 'mcaptcha', 'recaptcha', 'turnstile', 'fc', 'testcaptcha'] as const; export type CaptchaProvider = typeof supportedCaptchaProviders[number]; -export const captchaErrorCodes = { - invalidProvider: Symbol('invalidProvider'), - invalidParameters: Symbol('invalidParameters'), - noResponseProvided: Symbol('noResponseProvided'), - requestFailed: Symbol('requestFailed'), - verificationFailed: Symbol('verificationFailed'), - unknown: Symbol('unknown'), -} as const; -export type CaptchaErrorCode = typeof captchaErrorCodes[keyof typeof captchaErrorCodes]; - export type CaptchaSetting = { provider: CaptchaProvider; hcaptcha: { diff --git a/packages/backend/src/core/ChannelFollowingService.ts b/packages/backend/src/core/ChannelFollowingService.ts index 430711fef1..1c854b83fb 100644 --- a/packages/backend/src/core/ChannelFollowingService.ts +++ b/packages/backend/src/core/ChannelFollowingService.ts @@ -9,16 +9,13 @@ import { DI } from '@/di-symbols.js'; import type { ChannelFollowingsRepository } from '@/models/_.js'; import { MiChannel } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; -import { GlobalEvents, GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js'; +import { GlobalEventService } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; import type { MiLocalUser } from '@/models/User.js'; -import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; -import { InternalEventService } from './InternalEventService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; @Injectable() -export class ChannelFollowingService implements OnModuleInit { - public userFollowingChannelsCache: QuantumKVCache>; - +export class ChannelFollowingService { constructor( @Inject(DI.redis) private redisClient: Redis.Redis, @@ -29,21 +26,7 @@ export class ChannelFollowingService implements OnModuleInit { private idService: IdService, private globalEventService: GlobalEventService, private readonly internalEventService: InternalEventService, - ) { - this.userFollowingChannelsCache = new QuantumKVCache>(this.internalEventService, 'userFollowingChannels', { - lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.channelFollowingsRepository.find({ - where: { followerId: key }, - select: ['followeeId'], - }).then(xs => new Set(xs.map(x => x.followeeId))), - }); - - this.internalEventService.on('followChannel', this.onMessage); - this.internalEventService.on('unfollowChannel', this.onMessage); - } - - onModuleInit() { - } + ) {} @bindThis public async follow( @@ -56,7 +39,7 @@ export class ChannelFollowingService implements OnModuleInit { followeeId: targetChannel.id, }); - this.globalEventService.publishInternalEvent('followChannel', { + await this.internalEventService.emit('followChannel', { userId: requestUser.id, channelId: targetChannel.id, }); @@ -72,37 +55,9 @@ export class ChannelFollowingService implements OnModuleInit { followeeId: targetChannel.id, }); - this.globalEventService.publishInternalEvent('unfollowChannel', { + await this.internalEventService.emit('unfollowChannel', { userId: requestUser.id, channelId: targetChannel.id, }); } - - @bindThis - private async onMessage(body: InternalEventTypes[E], type: E): Promise { - { - switch (type) { - case 'followChannel': { - await this.userFollowingChannelsCache.delete(body.userId); - break; - } - case 'unfollowChannel': { - await this.userFollowingChannelsCache.delete(body.userId); - break; - } - } - } - } - - @bindThis - public dispose(): void { - this.internalEventService.off('followChannel', this.onMessage); - this.internalEventService.off('unfollowChannel', this.onMessage); - this.userFollowingChannelsCache.dispose(); - } - - @bindThis - public onApplicationShutdown(signal?: string | undefined): void { - this.dispose(); - } } diff --git a/packages/backend/src/core/ChatService.ts b/packages/backend/src/core/ChatService.ts index 76f4e0dbdd..cdecd41726 100644 --- a/packages/backend/src/core/ChatService.ts +++ b/packages/backend/src/core/ChatService.ts @@ -28,6 +28,7 @@ import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { emojiRegex } from '@/misc/emoji-regex.js'; import { NotificationService } from '@/core/NotificationService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; +import { TimeService } from '@/global/TimeService.js'; const MAX_ROOM_MEMBERS = 30; const MAX_REACTIONS_PER_MESSAGE = 100; @@ -91,6 +92,7 @@ export class ChatService { private userFollowingService: UserFollowingService, private customEmojiService: CustomEmojiService, private moderationLogService: ModerationLogService, + private readonly timeService: TimeService, ) { } @@ -225,7 +227,7 @@ export class ChatService { // 3秒経っても既読にならなかったらイベント発行 if (this.userEntityService.isLocalUser(toUser)) { - setTimeout(async () => { + this.timeService.startTimer(async () => { const marker = await this.redisClient.get(`newUserChatMessageExists:${toUser.id}:${fromUser.id}`); if (marker == null) return; // 既読 @@ -285,7 +287,7 @@ export class ChatService { redisPipeline.exec(); // 3秒経っても既読にならなかったらイベント発行 - setTimeout(async () => { + this.timeService.startTimer(async () => { const redisPipeline = this.redisClient.pipeline(); for (const membership of membershipsOtherThanMe) { redisPipeline.get(`newRoomChatMessageExists:${membership.userId}:${toRoom.id}`); @@ -820,7 +822,7 @@ export class ChatService { reaction = normalizeEmojiString(reaction_); } else { const name = custom[1]; - const emoji = (await this.customEmojiService.localEmojisCache.fetch()).get(name); + const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(name); if (emoji == null) { throw new Error('no such emoji'); diff --git a/packages/backend/src/core/ClipService.ts b/packages/backend/src/core/ClipService.ts index 929a9db064..35b962cc04 100644 --- a/packages/backend/src/core/ClipService.ts +++ b/packages/backend/src/core/ClipService.ts @@ -12,6 +12,7 @@ import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import type { MiLocalUser } from '@/models/User.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class ClipService { @@ -33,6 +34,7 @@ export class ClipService { private roleService: RoleService, private idService: IdService, + private readonly timeService: TimeService, ) { } @@ -125,7 +127,7 @@ export class ClipService { } this.clipsRepository.update(clip.id, { - lastClippedAt: new Date(), + lastClippedAt: this.timeService.date, }); this.notesRepository.increment({ id: noteId }, 'clippedCount', 1); diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index f818a65ff8..11119c52a3 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -15,11 +15,10 @@ import { SystemWebhookService } from '@/core/SystemWebhookService.js'; import { UserSearchService } from '@/core/UserSearchService.js'; import { WebhookTestService } from '@/core/WebhookTestService.js'; import { FlashService } from '@/core/FlashService.js'; -import { TimeService } from '@/core/TimeService.js'; -import { EnvService } from '@/core/EnvService.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { ApLogService } from '@/core/ApLogService.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; +import { InstanceStatsService } from '@/core/InstanceStatsService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import { AccountMoveService } from './AccountMoveService.js'; import { AccountUpdateService } from './AccountUpdateService.js'; @@ -42,7 +41,6 @@ import { HttpRequestService } from './HttpRequestService.js'; import { IdService } from './IdService.js'; import { ImageProcessingService } from './ImageProcessingService.js'; import { SystemAccountService } from './SystemAccountService.js'; -import { InternalEventService } from './InternalEventService.js'; import { InternalStorageService } from './InternalStorageService.js'; import { MetaService } from './MetaService.js'; import { MfmService } from './MfmService.js'; @@ -159,12 +157,11 @@ import { ApPersonService } from './activitypub/models/ApPersonService.js'; import { ApQuestionService } from './activitypub/models/ApQuestionService.js'; import { QueueModule } from './QueueModule.js'; import { QueueService } from './QueueService.js'; -import { LoggerService } from './LoggerService.js'; import { SponsorsService } from './SponsorsService.js'; import type { Provider } from '@nestjs/common'; +import { GlobalModule } from '@/GlobalModule.js'; //#region 文字列ベースでのinjection用(循環参照対応のため) -const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService }; const $AbuseReportService: Provider = { provide: 'AbuseReportService', useExisting: AbuseReportService }; const $AbuseReportNotificationService: Provider = { provide: 'AbuseReportNotificationService', useExisting: AbuseReportNotificationService }; const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisting: AccountMoveService }; @@ -188,7 +185,6 @@ const $HashtagService: Provider = { provide: 'HashtagService', useExisting: Hash const $HttpRequestService: Provider = { provide: 'HttpRequestService', useExisting: HttpRequestService }; const $IdService: Provider = { provide: 'IdService', useExisting: IdService }; const $ImageProcessingService: Provider = { provide: 'ImageProcessingService', useExisting: ImageProcessingService }; -const $InternalEventService: Provider = { provide: 'InternalEventService', useExisting: InternalEventService }; const $InternalStorageService: Provider = { provide: 'InternalStorageService', useExisting: InternalStorageService }; const $MetaService: Provider = { provide: 'MetaService', useExisting: MetaService }; const $MfmService: Provider = { provide: 'MfmService', useExisting: MfmService }; @@ -239,8 +235,7 @@ const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService', const $ChatService: Provider = { provide: 'ChatService', useExisting: ChatService }; const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService }; const $ReversiService: Provider = { provide: 'ReversiService', useExisting: ReversiService }; -const $TimeService: Provider = { provide: 'TimeService', useExisting: TimeService }; -const $EnvService: Provider = { provide: 'EnvService', useExisting: EnvService }; +const $InstanceStatsService = { provide: 'InstanceStatsService', useExisting: InstanceStatsService }; const $NoteVisibilityService: Provider = { provide: 'NoteVisibilityService', useExisting: NoteVisibilityService }; const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService }; @@ -322,10 +317,10 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp @Module({ imports: [ + GlobalModule, QueueModule, ], providers: [ - LoggerService, AbuseReportService, AbuseReportNotificationService, AccountMoveService, @@ -349,7 +344,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp HttpRequestService, IdService, ImageProcessingService, - InternalEventService, InternalStorageService, MetaService, MfmService, @@ -400,8 +394,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp ChatService, RegistryApiService, ReversiService, - TimeService, - EnvService, + InstanceStatsService, NoteVisibilityService, ChartLoggerService, @@ -482,7 +475,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp SponsorsService, //#region 文字列ベースでのinjection用(循環参照対応のため) - $LoggerService, $AbuseReportService, $AbuseReportNotificationService, $AccountMoveService, @@ -506,7 +498,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $HttpRequestService, $IdService, $ImageProcessingService, - $InternalEventService, $InternalStorageService, $MetaService, $MfmService, @@ -557,8 +548,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $ChatService, $RegistryApiService, $ReversiService, - $TimeService, - $EnvService, + $InstanceStatsService, $NoteVisibilityService, $ChartLoggerService, @@ -640,7 +630,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp ], exports: [ QueueModule, - LoggerService, AbuseReportService, AbuseReportNotificationService, AccountMoveService, @@ -664,7 +653,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp HttpRequestService, IdService, ImageProcessingService, - InternalEventService, InternalStorageService, MetaService, MfmService, @@ -715,8 +703,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp ChatService, RegistryApiService, ReversiService, - TimeService, - EnvService, + InstanceStatsService, NoteVisibilityService, FederationChart, @@ -796,7 +783,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp SponsorsService, //#region 文字列ベースでのinjection用(循環参照対応のため) - $LoggerService, $AbuseReportService, $AbuseReportNotificationService, $AccountMoveService, @@ -820,7 +806,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $HttpRequestService, $IdService, $ImageProcessingService, - $InternalEventService, $InternalStorageService, $MetaService, $MfmService, @@ -870,8 +855,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $ChatService, $RegistryApiService, $ReversiService, - $TimeService, - $EnvService, + $InstanceStatsService, $NoteVisibilityService, $FederationChart, diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index 2e4eddf797..dff6560378 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -5,7 +5,7 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { In, IsNull } from 'typeorm'; +import { In, IsNull, Not } from 'typeorm'; import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { IdService } from '@/core/IdService.js'; @@ -14,12 +14,29 @@ import { bindThis } from '@/decorators.js'; import { DI } from '@/di-symbols.js'; import { MemoryKVCache, RedisSingleCache } from '@/misc/cache.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; -import type { DriveFilesRepository, EmojisRepository, MiRole, MiUser } from '@/models/_.js'; +import type { DriveFilesRepository, EmojisRepository, MiRole, MiUser, MiDriveFile, NotesRepository } from '@/models/_.js'; import type { MiEmoji } from '@/models/Emoji.js'; import type { Serialized } from '@/types.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import type { Config } from '@/config.js'; -import { DriveService } from './DriveService.js'; +import { DriveService } from '@/core/DriveService.js'; +import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js'; +import { TimeService } from '@/global/TimeService.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import { isRetryableSymbol } from '@/misc/is-retryable-error.js'; +import type Logger from '@/logger.js'; +import { KeyNotFoundError } from '@/misc/errors/KeyNotFoundError.js'; + +// TODO move to sk-types.d.ts when merged +type MinEntity = Omit> & { + [K in NullableProps]?: T[K] | undefined; +}; +type SemiPartial = Omit & { + [Key in P]?: T[Key] | undefined; +}; +type NullableProps = { + [K in keyof T]: null extends T[K] ? K : never; +}[keyof T]; const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/; @@ -60,9 +77,15 @@ export const fetchEmojisSortKeys = [ export type FetchEmojisSortKeys = typeof fetchEmojisSortKeys[number]; @Injectable() -export class CustomEmojiService implements OnApplicationShutdown { - private emojisCache: MemoryKVCache; - public localEmojisCache: RedisSingleCache>; +export class CustomEmojiService { + // id -> MiEmoji + public readonly emojisByIdCache: ManagedQuantumKVCache; + + // key ("name host") -> MiEmoji (for remote emojis) + // key ("name") -> MiEmoji (for local emojis) + public readonly emojisByKeyCache: ManagedQuantumKVCache; + + private readonly logger: Logger; constructor( @Inject(DI.redis) @@ -73,29 +96,55 @@ export class CustomEmojiService implements OnApplicationShutdown { private emojisRepository: EmojisRepository, @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, + + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + private utilityService: UtilityService, private idService: IdService, private emojiEntityService: EmojiEntityService, private moderationLogService: ModerationLogService, private globalEventService: GlobalEventService, private driveService: DriveService, - ) { - this.emojisCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h + private readonly timeService: TimeService, - this.localEmojisCache = new RedisSingleCache>(this.redisClient, 'localEmojis', { - lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60 * 3, // 3m - fetcher: () => this.emojisRepository.find({ where: { host: IsNull() } }).then(emojis => new Map(emojis.map(emoji => [emoji.name, emoji]))), - toRedisConverter: (value) => JSON.stringify(Array.from(value.values())), - fromRedisConverter: (value) => { - return new Map(JSON.parse(value).map((x: Serialized) => [x.name, { - ...x, - updatedAt: x.updatedAt ? new Date(x.updatedAt) : null, - }])); + cacheManagementService: CacheManagementService, + loggerService: LoggerService, + ) { + this.logger = loggerService.getLogger('custom-emoji'); + + this.emojisByIdCache = cacheManagementService.createQuantumKVCache('emojisById', { + lifetime: 1000 * 60 * 60, // 1h + fetcher: async (id) => await this.emojisRepository.findOneByOrFail({ id }), + optionalFetcher: async (id) => await this.emojisRepository.findOneBy({ id }), + bulkFetcher: async (ids) => { + const emojis = await this.emojisRepository.findBy({ id: In(ids) }); + return emojis.map(emoji => [emoji.id, emoji]); + }, + }); + + this.emojisByKeyCache = cacheManagementService.createQuantumKVCache('emojisByKey', { + lifetime: 1000 * 60 * 60, // 1h + fetcher: async (key) => { + const { host, name } = decodeEmojiKey(key); + return await this.emojisRepository.findOneByOrFail({ host: host ?? IsNull(), name }); + }, + optionalFetcher: async (key) => { + const { host, name } = decodeEmojiKey(key); + return await this.emojisRepository.findOneBy({ host: host ?? IsNull(), name }); + }, + bulkFetcher: async (keys) => { + const queries = keys.map(key => { + const { host, name } = decodeEmojiKey(key); + return { host: host ?? IsNull(), name }; + }); + const emojis = await this.emojisRepository.findBy(queries); + return emojis.map(emoji => [encodeEmojiKey(emoji), emoji]); }, }); } + /** @deprecated use createEmoji for new code */ @bindThis public async add(data: { originalUrl: string; @@ -110,31 +159,39 @@ export class CustomEmojiService implements OnApplicationShutdown { localOnly: boolean; roleIdsThatCanBeUsedThisEmojiAsReaction: MiRole['id'][]; }, moderator?: MiUser): Promise { - const emoji = await this.emojisRepository.insertOne({ - id: this.idService.gen(), - updatedAt: new Date(), - name: data.name, - category: data.category, - host: data.host, - aliases: data.aliases, - originalUrl: data.originalUrl, - publicUrl: data.publicUrl, - type: data.fileType, - license: data.license, - isSensitive: data.isSensitive, - localOnly: data.localOnly, - roleIdsThatCanBeUsedThisEmojiAsReaction: data.roleIdsThatCanBeUsedThisEmojiAsReaction, - }); + return await this.createEmoji(data, { moderator }); + } - if (data.host == null) { - this.localEmojisCache.refresh(); + public async createEmoji( + data: SemiPartial, 'id' | 'updatedAt' | 'aliases' | 'roleIdsThatCanBeUsedThisEmojiAsReaction'>, + opts?: { moderator?: { id: string } }, + ): Promise { + // Set defaults + data.id ??= this.idService.gen(); + data.updatedAt ??= this.timeService.date; + data.aliases ??= []; + data.roleIdsThatCanBeUsedThisEmojiAsReaction ??= []; - this.globalEventService.publishBroadcastStream('emojiAdded', { - emoji: await this.emojiEntityService.packDetailed(emoji.id), + // Add to logs + this.logger.info(`Creating emoji name=${data.name} host=${data.host}...`); + + // Add to database + await this.emojisRepository.insert(data); + + // Add to cache + const emoji = await this.emojisByIdCache.fetch(data.id); + const emojiKey = encodeEmojiKey({ name: emoji.name, host: emoji.host }); + this.emojisByIdCache.add(emojiKey, emoji); // This is a new entity, so we can use add() which does not emit sync events. + + if (emoji.host == null) { + // Add to clients + await this.globalEventService.publishBroadcastStream('emojiAdded', { + emoji: await this.emojiEntityService.packDetailed(emoji), }); - if (moderator) { - this.moderationLogService.log(moderator, 'addCustomEmoji', { + // Add to mod logs + if (opts?.moderator) { + await this.moderationLogService.log(opts.moderator, 'addCustomEmoji', { emojiId: emoji.id, emoji: emoji, }); @@ -144,6 +201,7 @@ export class CustomEmojiService implements OnApplicationShutdown { return emoji; } + /** @deprecated Use updateEmoji for new code */ @bindThis public async update(data: ( { id: MiEmoji['id'], name?: string; } | { name: string; id?: MiEmoji['id'], } @@ -161,91 +219,161 @@ export class CustomEmojiService implements OnApplicationShutdown { null | 'NO_SUCH_EMOJI' | 'SAME_NAME_EMOJI_EXISTS' - > { - const emoji = data.id - ? await this.getEmojiById(data.id) - : await this.getEmojiByName(data.name!); - if (emoji === null) return 'NO_SUCH_EMOJI'; - const id = emoji.id; + > { + try { + const criteria = data.id + ? { id: data.id as string } + : { name: data.name as string, host: null }; - // IDと絵文字名が両方指定されている場合は絵文字名の変更を行うため重複チェックが必要 - const doNameUpdate = data.id && data.name && (data.name !== emoji.name); + const updates = { + ...data, + id: undefined, + host: undefined, + }; + + const opts = { + moderator, + }; + + await this.updateEmoji(criteria, updates, opts); + return null; + } catch (err) { + if (err instanceof KeyNotFoundError) return 'NO_SUCH_EMOJI'; + if (err instanceof DuplicateEmojiError) return 'SAME_NAME_EMOJI_EXISTS'; + throw err; + } + } + + @bindThis + public async updateEmoji( + criteria: { id: string } | { name: string, host: string | null }, + data: Omit, 'id' | 'host'>, + opts?: { moderator?: { id: string } }, + ): Promise { + const emoji = 'id' in criteria + ? await this.emojisByIdCache.fetch(criteria.id) + : await this.emojisByKeyCache.fetch(encodeEmojiKey(criteria)); + + // Update the system logs + this.logger.info(`Updating emoji name=${emoji.name} host=${emoji.host}...`); + + // If changing the name, then make sure we don't have a conflict. + const doNameUpdate = data.name !== undefined && data.name !== emoji.name; if (doNameUpdate) { - const isDuplicate = await this.checkDuplicate(data.name!); - if (isDuplicate) return 'SAME_NAME_EMOJI_EXISTS'; + const isDuplicate = await this.checkDuplicate(data.name as string, emoji.host); + if (isDuplicate) throw new DuplicateEmojiError(data.name as string, emoji.host); } - await this.emojisRepository.update(emoji.id, { - updatedAt: new Date(), - name: data.name, - category: data.category, - aliases: data.aliases, - license: data.license, - isSensitive: data.isSensitive, - localOnly: data.localOnly, - originalUrl: data.originalUrl, - publicUrl: data.publicUrl, - type: data.fileType, - roleIdsThatCanBeUsedThisEmojiAsReaction: data.roleIdsThatCanBeUsedThisEmojiAsReaction ?? undefined, - }); + // Make sure we always set the updated date! + data.updatedAt ??= this.timeService.date; - this.localEmojisCache.refresh(); + // Update the database + await this.emojisRepository.update({ id: emoji.id }, data); - // If we're changing the file, then we need to delete the old one - if (data.originalUrl != null && data.originalUrl !== emoji.originalUrl) { - const oldFile = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); - const newFile = await this.driveFilesRepository.findOneBy({ url: data.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); + // Update the caches + const updated = await this.emojisByIdCache.refresh(emoji.id); + const updatedKey = encodeEmojiKey({ name: emoji.name, host: emoji.host }); + await this.emojisByKeyCache.set(updatedKey, updated); - // But DON'T delete if this is the same file reference, otherwise we'll break the emoji! - if (oldFile && newFile && oldFile.id !== newFile.id) { - await this.driveService.deleteFile(oldFile, false, moderator ? moderator : undefined); - } + // Update the file + await this.updateEmojiFile(emoji, updated); + + // If it's a remote emoji, then we're done. + // The remaining logic applies only to local emojis. + if (updated.host != null) { + return updated; } - const packed = await this.emojiEntityService.packDetailed(emoji.id); - + // Update the clients if (!doNameUpdate) { - this.globalEventService.publishBroadcastStream('emojiUpdated', { + // If name is the same, then we can update in-place + const packed = await this.emojiEntityService.packDetailed(updated); + await this.globalEventService.publishBroadcastStream('emojiUpdated', { emojis: [packed], }); } else { - this.globalEventService.publishBroadcastStream('emojiDeleted', { - emojis: [await this.emojiEntityService.packDetailed(emoji)], + // If name has changed, we need to delete and recreate + const [oldPacked, newPacked] = await Promise.all([ + this.emojiEntityService.packDetailed(emoji), + this.emojiEntityService.packDetailed(updated), + ]); + + await this.globalEventService.publishBroadcastStream('emojiDeleted', { + emojis: [oldPacked], }); - this.globalEventService.publishBroadcastStream('emojiAdded', { - emoji: packed, + await this.globalEventService.publishBroadcastStream('emojiAdded', { + emoji: newPacked, }); } - if (moderator) { - const updated = await this.emojisRepository.findOneByOrFail({ id: id }); - this.moderationLogService.log(moderator, 'updateCustomEmoji', { + // Update the mod logs + if (opts?.moderator) { + await this.moderationLogService.log(opts.moderator, 'updateCustomEmoji', { emojiId: emoji.id, before: emoji, after: updated, }); } - return null; + + return updated; + } + + @bindThis + private async updateEmojiFile(before: MiEmoji, after: MiEmoji, moderator?: { id: string }): Promise { + // Nothing to do + if (after.originalUrl === before.originalUrl) { + return; + } + + // If we're changing the file, then we need to delete the old one. + const [oldFile, newFile] = await Promise.all([ + this.driveFilesRepository.findOneBy({ url: before.originalUrl, userHost: before.host ?? IsNull() }), + this.driveFilesRepository.findOneBy({ url: after.originalUrl, userHost: after.host ?? IsNull() }), + ]); + + // But DON'T delete if this is the same file reference, otherwise we'll break the emoji! + if (!oldFile || !newFile || oldFile.id === newFile.id) { + return; + } + + await this.safeDeleteEmojiFile(before, oldFile, moderator); + } + + @bindThis + private async safeDeleteEmojiFile(emoji: MiEmoji, file: MiDriveFile, moderator?: { id: string }): Promise { + const [hasNoteReferences, hasEmojiReferences] = await Promise.all([ + // Any note using this file ID is a reference. + this.notesRepository + .createQueryBuilder('note') + .where(':fileId <@ note.fileIds', { fileId: file.id }) + .getExists(), + // Any *other* emoji using this file URL is a reference. + this.emojisRepository.existsBy({ + originalUrl: file.url, + id: Not(emoji.id), + }), + ]); + + if (hasNoteReferences) { + this.logger.debug(`Not removing old file ${file.id} (${file.url}) - file is referenced by one or more notes.`); + } else if (hasEmojiReferences) { + this.logger.debug(`Not removing old file ${file.id} (${file.url}) - file is reference by another emoji.`); + } else { + this.logger.info(`Removing old file ${file.id} (${file.url}).`); + await this.driveService.deleteFile(file, false, moderator); + } } @bindThis public async addAliasesBulk(ids: MiEmoji['id'][], aliases: string[]) { - const emojis = await this.emojisRepository.findBy({ - id: In(ids), - }); - - for (const emoji of emojis) { - await this.emojisRepository.update(emoji.id, { - updatedAt: new Date(), - aliases: [...new Set(emoji.aliases.concat(aliases))], - }); - } - - this.localEmojisCache.refresh(); - - this.globalEventService.publishBroadcastStream('emojiUpdated', { - emojis: await this.emojiEntityService.packDetailedMany(ids), + await this.bulkUpdateEmojis(ids, async emojis => { + for (const emoji of emojis) { + await this.emojisRepository.update(emoji.id, { + updatedAt: this.timeService.date, + aliases: [...new Set(emoji.aliases.concat(aliases))], + }); + } }); } @@ -254,34 +382,22 @@ export class CustomEmojiService implements OnApplicationShutdown { await this.emojisRepository.update({ id: In(ids), }, { - updatedAt: new Date(), + updatedAt: this.timeService.date, aliases: aliases, }); - this.localEmojisCache.refresh(); - - this.globalEventService.publishBroadcastStream('emojiUpdated', { - emojis: await this.emojiEntityService.packDetailedMany(ids), - }); + await this.bulkUpdateEmojis(ids); } @bindThis public async removeAliasesBulk(ids: MiEmoji['id'][], aliases: string[]) { - const emojis = await this.emojisRepository.findBy({ - id: In(ids), - }); - - for (const emoji of emojis) { - await this.emojisRepository.update(emoji.id, { - updatedAt: new Date(), - aliases: emoji.aliases.filter(x => !aliases.includes(x)), - }); - } - - this.localEmojisCache.refresh(); - - this.globalEventService.publishBroadcastStream('emojiUpdated', { - emojis: await this.emojiEntityService.packDetailedMany(ids), + await this.bulkUpdateEmojis(ids, async emojis => { + for (const emoji of emojis) { + await this.emojisRepository.update(emoji.id, { + updatedAt: this.timeService.date, + aliases: emoji.aliases.filter(x => !aliases.includes(x)), + }); + } }); } @@ -290,15 +406,11 @@ export class CustomEmojiService implements OnApplicationShutdown { await this.emojisRepository.update({ id: In(ids), }, { - updatedAt: new Date(), + updatedAt: this.timeService.date, category: category, }); - this.localEmojisCache.refresh(); - - this.globalEventService.publishBroadcastStream('emojiUpdated', { - emojis: await this.emojiEntityService.packDetailedMany(ids), - }); + await this.bulkUpdateEmojis(ids); } @bindThis @@ -306,71 +418,115 @@ export class CustomEmojiService implements OnApplicationShutdown { await this.emojisRepository.update({ id: In(ids), }, { - updatedAt: new Date(), + updatedAt: this.timeService.date, license: license, }); - this.localEmojisCache.refresh(); + await this.bulkUpdateEmojis(ids); + } - this.globalEventService.publishBroadcastStream('emojiUpdated', { - emojis: await this.emojiEntityService.packDetailedMany(ids), + @bindThis + private async bulkUpdateEmojis(ids: MiEmoji['id'][], updater?: (emojis: readonly MiEmoji[]) => Promise): Promise { + // Update the database + if (updater) { + const emojis = await this.emojisByIdCache.fetchMany(ids); + await updater(emojis.values); + } + + // Update the caches + const updated = await this.emojisByIdCache.refreshMany(ids); + const keyUpdates = updated.values.map(emoji => [encodeEmojiKey(emoji), emoji] as const); + await this.emojisByKeyCache.setMany(keyUpdates); + + // Update the clients + await this.globalEventService.publishBroadcastStream('emojiUpdated', { + emojis: await this.emojiEntityService.packDetailedMany(updated.values), }); } @bindThis - public async delete(id: MiEmoji['id'], moderator?: MiUser) { - const emoji = await this.emojisRepository.findOneByOrFail({ id: id }); + public async delete(id: MiEmoji['id'], moderator?: { id: string }) { + const emoji = await this.emojisByIdCache.fetch(id); - await this.emojisRepository.delete(emoji.id); + await Promise.all([ + this.emojisRepository.delete(emoji.id), + this.emojisByIdCache.delete(emoji.id), + this.emojisByKeyCache.delete(encodeEmojiKey(emoji)), + ]); - this.localEmojisCache.refresh(); - - const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); + const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ?? IsNull() }); if (file) { - await this.driveService.deleteFile(file, false, moderator ? moderator : undefined); + await this.safeDeleteEmojiFile(emoji, file, moderator); } - this.globalEventService.publishBroadcastStream('emojiDeleted', { - emojis: [await this.emojiEntityService.packDetailed(emoji)], - }); - - if (moderator) { - this.moderationLogService.log(moderator, 'deleteCustomEmoji', { - emojiId: emoji.id, - emoji: emoji, + if (emoji.host == null) { + await this.globalEventService.publishBroadcastStream('emojiDeleted', { + emojis: [await this.emojiEntityService.packDetailed(emoji)], }); - } - } - - @bindThis - public async deleteBulk(ids: MiEmoji['id'][], moderator?: MiUser) { - const emojis = await this.emojisRepository.findBy({ - id: In(ids), - }); - - for (const emoji of emojis) { - await this.emojisRepository.delete(emoji.id); - - const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); - - if (file) { - await this.driveService.deleteFile(file, false, moderator ? moderator : undefined); - } if (moderator) { - this.moderationLogService.log(moderator, 'deleteCustomEmoji', { + await this.moderationLogService.log(moderator, 'deleteCustomEmoji', { emojiId: emoji.id, emoji: emoji, }); } } + } - this.localEmojisCache.refresh(); + @bindThis + public async deleteBulk(ids: MiEmoji['id'][], moderator?: MiUser) { + const emojis = await this.emojisByIdCache.fetchMany(ids); - this.globalEventService.publishBroadcastStream('emojiDeleted', { - emojis: await this.emojiEntityService.packDetailedMany(emojis), - }); + const filesQueries = emojis.values.map(emoji => ({ + url: emoji.originalUrl, + userHost: emoji.host ?? IsNull(), + })); + const files = await this.driveFilesRepository.findBy(filesQueries); + + const emojiFiles = emojis.values + .map(emoji => { + const file = files.find(file => file.url === emoji.originalUrl && file.userHost === emoji.host); + return [emoji, file]; + }) + .filter(ef => ef[1] != null) as [MiEmoji, MiDriveFile][]; + + const localDeleted = emojis.values.filter(emoji => emoji.host == null); + const deletedKeys = emojis.values.map(emoji => encodeEmojiKey(emoji)); + await Promise.all([ + // Delete from database + this.emojisRepository.delete({ id: In(ids) }), + this.emojisByIdCache.deleteMany(ids), + + // Delete from cache + this.emojisByKeyCache.deleteMany(deletedKeys), + + // Delete from clients + localDeleted.length > 0 + ? this.emojiEntityService.packDetailedMany(localDeleted).then(async packed => { + await this.globalEventService.publishBroadcastStream('emojiDeleted', { + emojis: packed, + }); + }) + : null, + + // Delete from mod logs + localDeleted.length > 0 && moderator != null + ? Promise.all(localDeleted.map(async emoji => { + await this.moderationLogService.log(moderator, 'deleteCustomEmoji', { + emojiId: emoji.id, + emoji: emoji, + }); + })) + : null, + + // Delete from drive + emojiFiles.length > 0 + ? Promise.all(emojiFiles.map(async ([emoji, file]) => { + await this.safeDeleteEmojiFile(emoji, file, moderator); + })) + : null, + ]); } @bindThis @@ -407,18 +563,10 @@ export class CustomEmojiService implements OnApplicationShutdown { */ @bindThis public async populateEmoji(emojiName: string, noteUserHost: string | null): Promise { - const { name, host } = this.parseEmojiStr(emojiName, noteUserHost); - if (name == null) return null; - if (host == null) return null; + const emojiKey = this.translateEmojiKey(emojiName, noteUserHost); + if (emojiKey == null) return null; - const newHost = host === this.config.host ? null : host; - - const queryOrNull = async () => (await this.emojisRepository.findOneBy({ - name, - host: newHost ?? IsNull(), - })) ?? null; - - const emoji = await this.emojisCache.fetch(`${name} ${host}`, queryOrNull); + const emoji = await this.emojisByKeyCache.fetchMaybe(emojiKey); if (emoji == null) return null; return emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ） @@ -440,47 +588,45 @@ export class CustomEmojiService implements OnApplicationShutdown { return res; } + @bindThis + private translateEmojiKey(emojiName: string, noteUserHost: string | null): string | null { + const { name, host } = this.parseEmojiStr(emojiName, noteUserHost); + if (name == null) return null; + if (host == null) return null; + + const newHost = host === this.config.host ? null : host; + return encodeEmojiKey({ name, host: newHost }); + } + /** * 与えられた絵文字のリストをデータベースから取得し、キャッシュに追加します */ @bindThis public async prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise { - const notCachedEmojis = emojis.filter(emoji => this.emojisCache.get(`${emoji.name} ${emoji.host}`) == null); - const emojisQuery: any[] = []; - const hosts = new Set(notCachedEmojis.map(e => e.host)); - for (const host of hosts) { - if (host == null) continue; - emojisQuery.push({ - name: In(notCachedEmojis.filter(e => e.host === host).map(e => e.name)), - host: host, - }); - } - const _emojis = emojisQuery.length > 0 ? await this.emojisRepository.find({ - where: emojisQuery, - select: ['name', 'host', 'originalUrl', 'publicUrl'], - }) : []; - for (const emoji of _emojis) { - this.emojisCache.set(`${emoji.name} ${emoji.host}`, emoji); - } + const emojiKeys = emojis.map(emoji => encodeEmojiKey(emoji)); + await this.emojisByKeyCache.fetchMany(emojiKeys); } /** * ローカル内の絵文字に重複がないかチェックします * @param name 絵文字名 + * @param host Emoji hostname */ @bindThis - public checkDuplicate(name: string): Promise { - return this.emojisRepository.exists({ where: { name, host: IsNull() } }); + public async checkDuplicate(name: string, host: string | null = null): Promise { + const emoji = await this.getEmojiByName(name, host); + return emoji != null; } @bindThis - public getEmojiById(id: string): Promise { - return this.emojisRepository.findOneBy({ id }); + public async getEmojiById(id: string): Promise { + return await this.emojisByIdCache.fetchMaybe(id) ?? null; } @bindThis - public getEmojiByName(name: string): Promise { - return this.emojisRepository.findOneBy({ name, host: IsNull() }); + public async getEmojiByName(name: string, host: string | null = null): Promise { + const emojiKey = encodeEmojiKey({ name, host }); + return await this.emojisByKeyCache.fetchMaybe(emojiKey) ?? null; } @bindThis @@ -627,14 +773,97 @@ export class CustomEmojiService implements OnApplicationShutdown { allPages: Math.ceil(count / limit), }; } +} - @bindThis - public dispose(): void { - this.emojisCache.dispose(); - } +export class InvalidEmojiError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; - @bindThis - public onApplicationShutdown(signal?: string | undefined): void { - this.dispose(); + public readonly [isRetryableSymbol] = false; +} + +export class InvalidEmojiKeyError extends InvalidEmojiError { + constructor( + public readonly key: string, + message?: string, + ) { + const actualMessage = message + ? `Invalid emoji key "${key}": ${message}` + : `Invalid emoji key "${key}".`; + super(actualMessage); } } + +export class InvalidEmojiNameError extends InvalidEmojiError { + constructor( + public readonly name: string, + message?: string, + ) { + const actualMessage = message + ? `Invalid emoji name "${name}": ${message}` + : `Invalid emoji name "${name}".`; + super(actualMessage); + } +} + +export class InvalidEmojiHostError extends InvalidEmojiError { + constructor( + public readonly host: string | null, + message?: string, + ) { + const hostString = host == null ? 'null' : `"${host}"`; + const actualMessage = message + ? `Invalid emoji name ${hostString}: ${message}` + : `Invalid emoji name ${hostString}.`; + super(actualMessage); + } +} + +export class DuplicateEmojiError extends InvalidEmojiError { + constructor( + public readonly name: string, + public readonly host: string | null, + message?: string, + ) { + const hostString = host == null ? 'null' : `"${host}"`; + const actualMessage = message + ? `Duplicate emoji name "${name}" for host ${hostString}: ${message}` + : `Duplicate emoji name "${name}" for host ${hostString}.`; + super(actualMessage); + } +} + +export function isValidEmojiName(name: string): boolean { + return name !== '' && !name.includes(' '); +} + +export function isValidEmojiHost(host: string): boolean { + return host !== '' && !host.includes(' '); +} + +// TODO unit tests +export function encodeEmojiKey(emoji: { name: string, host: string | null }): string { + if (emoji.name === '') throw new InvalidEmojiNameError(emoji.name, 'Name cannot be empty.'); + if (emoji.name.includes(' ')) throw new InvalidEmojiNameError(emoji.name, 'Name cannot contain a space.'); + + // Local emojis are just the name. + if (emoji.host == null) { + return emoji.name; + } + + if (emoji.host === '') throw new InvalidEmojiHostError(emoji.host, 'Host cannot be empty.'); + if (emoji.host.includes(' ')) throw new InvalidEmojiHostError(emoji.host, 'Host cannot contain a space.'); + return `${emoji.name} ${emoji.host}`; +} + +// TODO unit tests +export function decodeEmojiKey(key: string): { name: string, host: string | null } { + const match = key.match(/^([^ ]+)(?: ([^ ]+))?$/); + if (!match) { + throw new InvalidEmojiKeyError(key); + } + + const name = match[1]; + const host = match[2] || null; + return { name, host }; +} diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 9de68c597b..94289f237d 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -424,10 +424,10 @@ export class DriveService { if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read'; try { - if (this.bunnyService.usingBunnyCDN(this.meta)) { - await this.bunnyService.upload(this.meta, key, stream); + if (this.bunnyService.usingBunnyCDN()) { + await this.bunnyService.upload(key, stream); } else { - const result = await this.s3Service.upload(this.meta, params); + const result = await this.s3Service.upload(params); if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); } else { // AbortMultipartUploadCommandOutput @@ -739,7 +739,7 @@ export class DriveService { } @bindThis - public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: MiUser) { + public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: { id: string }) { if (file.storedInternal) { this.deleteLocalFile(file.accessKey!); @@ -766,8 +766,8 @@ export class DriveService { } @bindThis - public async deleteFileSync(file: MiDriveFile, isExpired = false, deleter?: MiUser) { - const promises = []; + public async deleteFileSync(file: MiDriveFile, isExpired = false, deleter?: { id: string }) { + const promises: Promise[] = []; if (file.storedInternal) { promises.push(this.deleteLocalFile(file.accessKey!)); @@ -797,7 +797,7 @@ export class DriveService { } @bindThis - private async deletePostProcess(file: MiDriveFile, isExpired = false, deleter?: MiUser) { + private async deletePostProcess(file: MiDriveFile, isExpired = false, deleter?: { id: string }) { // リモートファイル期限切れ削除後は直リンクにする if (isExpired && file.userHost !== null && file.uri != null) { this.driveFilesRepository.update(file.id, { @@ -843,15 +843,17 @@ export class DriveService { @bindThis public async deleteObjectStorageFile(key: string) { try { + if (this.bunnyService.usingBunnyCDN()) { + await this.bunnyService.delete(key); + return; + } + const param = { Bucket: this.meta.objectStorageBucket, Key: key, } as DeleteObjectCommandInput; - if (this.bunnyService.usingBunnyCDN(this.meta)) { - await this.bunnyService.delete(this.meta, key); - } else { - await this.s3Service.delete(this.meta, param); - } + + await this.s3Service.delete(param); } catch (err: any) { if (err.name === 'NoSuchKey') { this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`); diff --git a/packages/backend/src/core/EnvService.ts b/packages/backend/src/core/EnvService.ts deleted file mode 100644 index 8cc3b95735..0000000000 --- a/packages/backend/src/core/EnvService.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { Injectable } from '@nestjs/common'; - -/** - * Provides access to the process environment variables. - * This exists for testing purposes, so that a test can mock the environment without corrupting state for other tests. - */ -@Injectable() -export class EnvService { - /** - * Passthrough to process.env - */ - public get env() { - return process.env; - } -} diff --git a/packages/backend/src/core/FanoutTimelineEndpointService.ts b/packages/backend/src/core/FanoutTimelineEndpointService.ts index ddb0ddb7d2..6657a04dc9 100644 --- a/packages/backend/src/core/FanoutTimelineEndpointService.ts +++ b/packages/backend/src/core/FanoutTimelineEndpointService.ts @@ -276,7 +276,7 @@ export class FanoutTimelineEndpointService { // Fetch everything and populate users const [users, instances] = await Promise.all([ - this.cacheService.getUsers(usersToFetch), + this.cacheService.findUsersById(usersToFetch), this.federatedInstanceService.federatedInstanceCache.fetchMany(instancesToFetch).then(i => new Map(i)), ]); for (const [id, user] of Array.from(users)) { diff --git a/packages/backend/src/core/FanoutTimelineService.ts b/packages/backend/src/core/FanoutTimelineService.ts index 24999bf4da..1693fa686d 100644 --- a/packages/backend/src/core/FanoutTimelineService.ts +++ b/packages/backend/src/core/FanoutTimelineService.ts @@ -8,6 +8,7 @@ import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; export type FanoutTimelineName = ( // home timeline @@ -46,6 +47,7 @@ export class FanoutTimelineService { private redisForTimelines: Redis.Redis, private idService: IdService, + private readonly timeService: TimeService, ) { } @@ -53,7 +55,7 @@ export class FanoutTimelineService { public push(tl: FanoutTimelineName, id: string, maxlen: number, pipeline: Redis.ChainableCommander) { // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、 // 3分以内に投稿されたものでない場合、Redisにある最古のIDより新しい場合のみ追加する - if (this.idService.parse(id).date.getTime() > Date.now() - 1000 * 60 * 3) { + if (this.idService.parse(id).date.getTime() > this.timeService.now - 1000 * 60 * 3) { pipeline.lpush('list:' + tl, id); if (Math.random() < 0.1) { // 10%の確率でトリム pipeline.ltrim('list:' + tl, 0, maxlen - 1); diff --git a/packages/backend/src/core/FeaturedService.ts b/packages/backend/src/core/FeaturedService.ts index cabbb46504..e2475fff0a 100644 --- a/packages/backend/src/core/FeaturedService.ts +++ b/packages/backend/src/core/FeaturedService.ts @@ -9,6 +9,7 @@ import type { MiGalleryPost, MiNote, MiUser } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと export const GALLERY_POSTS_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと @@ -24,12 +25,13 @@ export class FeaturedService { private redisClient: Redis.Redis, // TODO: 専用のRedisサーバーを設定できるようにする private readonly roleService: RoleService, + private readonly timeService: TimeService, ) { } @bindThis private getCurrentWindow(windowRange: number): number { - const passed = new Date().getTime() - featuredEpoc; + const passed = this.timeService.now - featuredEpoc; return Math.floor(passed / windowRange); } diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index e549dbc93e..3e80ac2877 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -4,22 +4,24 @@ */ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; -import * as Redis from 'ioredis'; import { In } from 'typeorm'; -import type { InstancesRepository, MiMeta } from '@/models/_.js'; +import type { InstancesRepository } from '@/models/_.js'; +import type { MiMeta } from '@/models/Meta.js'; import type { MiInstance } from '@/models/Instance.js'; +import type { InternalEventTypes } from '@/core/GlobalEventService.js'; import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { bindThis } from '@/decorators.js'; +import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; import { diffArraysSimple } from '@/misc/diff-arrays.js'; -import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; +import { bindThis } from '@/decorators.js'; +import { TimeService } from '@/global/TimeService.js'; import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js'; @Injectable() export class FederatedInstanceService implements OnApplicationShutdown { - public readonly federatedInstanceCache: QuantumKVCache; + public readonly federatedInstanceCache: ManagedQuantumKVCache; constructor( @Inject(DI.instancesRepository) @@ -31,8 +33,12 @@ export class FederatedInstanceService implements OnApplicationShutdown { private utilityService: UtilityService, private idService: IdService, private readonly internalEventService: InternalEventService, + private readonly timeService: TimeService, + + cacheManagementService: CacheManagementService, ) { - this.federatedInstanceCache = new QuantumKVCache(this.internalEventService, 'federatedInstance', { + this.federatedInstanceCache = cacheManagementService.createQuantumKVCache('federatedInstance', { + // TODO can we increase this? lifetime: 1000 * 60 * 3, // 3 minutes fetcher: async key => { const host = this.utilityService.toPuny(key); @@ -43,7 +49,7 @@ export class FederatedInstanceService implements OnApplicationShutdown { .values({ id: this.idService.gen(), host, - firstRetrievedAt: new Date(), + firstRetrievedAt: this.timeService.date, isBlocked: this.utilityService.isBlockedHost(host), isSilenced: this.utilityService.isSilencedHost(host), isMediaSilenced: this.utilityService.isMediaSilencedHost(host), @@ -57,14 +63,15 @@ export class FederatedInstanceService implements OnApplicationShutdown { } return instance; }, + // optionalFetcher not needed bulkFetcher: async keys => { const hosts = keys.map(key => this.utilityService.toPuny(key)); const instances = await this.instancesRepository.findBy({ host: In(hosts) }); - return instances.map(i => [i.host, i]); + return instances.map(instance => [instance.host, instance]); }, }); - this.internalEventService.on('metaUpdated', this.onMetaUpdated); + this.internalEventService.on('metaUpdated', this.onMetaUpdated, { ignoreRemote: true }); } @bindThis @@ -83,7 +90,7 @@ export class FederatedInstanceService implements OnApplicationShutdown { .values({ id: this.idService.gen(), host, - firstRetrievedAt: new Date(), + firstRetrievedAt: this.timeService.date, isBlocked: this.utilityService.isBlockedHost(host), isSilenced: this.utilityService.isSilencedHost(host), isMediaSilenced: this.utilityService.isMediaSilencedHost(host), @@ -159,34 +166,31 @@ export class FederatedInstanceService implements OnApplicationShutdown { return instances.map(i => i[1]); } - // This gets fired *in each process* so don't do anything to trigger cache notifications! - private syncCache(before: MiMeta | undefined, after: MiMeta): void { - const changed = + @bindThis + private async onMetaUpdated(body: InternalEventTypes['metaUpdated']): Promise { + const { before, after } = body; + const changed = ( diffArraysSimple(before?.blockedHosts, after.blockedHosts) || diffArraysSimple(before?.silencedHosts, after.silencedHosts) || diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) || diffArraysSimple(before?.federationHosts, after.federationHosts) || - diffArraysSimple(before?.bubbleInstances, after.bubbleInstances); + diffArraysSimple(before?.bubbleInstances, after.bubbleInstances) + ); if (changed) { // We have to clear the whole thing, otherwise subdomains won't be synced. + // This gets fired in *each* process so don't do anything to trigger cache notifications! this.federatedInstanceCache.clear(); } } @bindThis - private async onMetaUpdated(body: { before?: MiMeta; after: MiMeta; }) { - this.syncCache(body.before, body.after); - } - - @bindThis - public dispose(): void { + public dispose() { this.internalEventService.off('metaUpdated', this.onMetaUpdated); - this.federatedInstanceCache.dispose(); } @bindThis - public onApplicationShutdown(signal?: string | undefined): void { + public onApplicationShutdown() { this.dispose(); } } diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 6fcfdfb596..d288c5d231 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -15,6 +15,7 @@ import { LoggerService } from '@/core/LoggerService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import { TimeService } from '@/global/TimeService.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import type { CheerioAPI } from 'cheerio/slim'; @@ -47,6 +48,8 @@ export class FetchInstanceMetadataService { private federatedInstanceService: FederatedInstanceService, @Inject(DI.redis) private redisClient: Redis.Redis, + + private readonly timeService: TimeService, ) { this.logger = this.loggerService.getLogger('metadata', 'cyan'); } @@ -84,7 +87,7 @@ export class FetchInstanceMetadataService { try { if (!force) { const _instance = await this.federatedInstanceService.fetchOrRegister(host); - const now = Date.now(); + const now = this.timeService.now; if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) { // unlock at the finally caluse return; @@ -110,7 +113,7 @@ export class FetchInstanceMetadataService { this.logger.debug(`Successfuly fetched metadata of ${instance.host}`); const updates = { - infoUpdatedAt: new Date(), + infoUpdatedAt: this.timeService.date, } as Record; if (info) { diff --git a/packages/backend/src/core/FileInfoService.ts b/packages/backend/src/core/FileInfoService.ts index 98fbfe5f23..9a10351c9b 100644 --- a/packages/backend/src/core/FileInfoService.ts +++ b/packages/backend/src/core/FileInfoService.ts @@ -143,7 +143,7 @@ export class FileInfoService { } @bindThis - public fixMime(mime: string | fileType.MimeType): string { + public fixMime(mime: string): string { // see https://github.com/misskey-dev/misskey/pull/10686 if (mime === 'audio/x-flac') { return 'audio/flac'; diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index c146811331..2be5a580da 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -20,12 +20,14 @@ import type { MiPage } from '@/models/Page.js'; import type { MiWebhook } from '@/models/Webhook.js'; import type { MiSystemWebhook } from '@/models/SystemWebhook.js'; import type { MiMeta } from '@/models/Meta.js'; -import { MiAvatarDecoration, MiChatMessage, MiChatRoom, MiReversiGame, MiRole, MiRoleAssignment } from '@/models/_.js'; +import type { MiAvatarDecoration, MiChatMessage, MiChatRoom, MiReversiGame, MiRole, MiRoleAssignment } from '@/models/_.js'; import type { Packed } from '@/misc/json-schema.js'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; -import { Serialized } from '@/types.js'; +import type { Serialized } from '@/types.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; import type Emitter from 'strict-event-emitter-types'; import type { EventEmitter } from 'events'; @@ -233,8 +235,12 @@ export interface InternalEventTypes { userChangeSuspendedState: { id: MiUser['id']; isSuspended: MiUser['isSuspended']; }; userChangeDeletedState: { id: MiUser['id']; isDeleted: MiUser['isDeleted']; }; userTokenRegenerated: { id: MiUser['id']; oldToken: string; newToken: string; }; + /** @deprecated Use userUpdated or usersUpdated instead */ remoteUserUpdated: { id: MiUser['id']; }; + /** @deprecated Use userUpdated or usersUpdated instead */ localUserUpdated: { id: MiUser['id']; }; + usersUpdated: { ids: MiUser['id'][]; }; + userUpdated: { id: MiUser['id']; }; follow: { followerId: MiUser['id']; followeeId: MiUser['id']; }; unfollow: { followerId: MiUser['id']; followeeId: MiUser['id']; }; blockingCreated: { blockerId: MiUser['id']; blockeeId: MiUser['id']; }; @@ -245,12 +251,12 @@ export interface InternalEventTypes { roleUpdated: MiRole; userRoleAssigned: MiRoleAssignment; userRoleUnassigned: MiRoleAssignment; - webhookCreated: MiWebhook; - webhookDeleted: MiWebhook; - webhookUpdated: MiWebhook; - systemWebhookCreated: MiSystemWebhook; - systemWebhookDeleted: MiSystemWebhook; - systemWebhookUpdated: MiSystemWebhook; + webhookCreated: { id: MiWebhook['id'] }; + webhookDeleted: { id: MiWebhook['id'] }; + webhookUpdated: { id: MiWebhook['id'] }; + systemWebhookCreated: { id: MiSystemWebhook['id'] }; + systemWebhookDeleted: { id: MiSystemWebhook['id'] }; + systemWebhookUpdated: { id: MiSystemWebhook['id'] }; antennaCreated: MiAntenna; antennaDeleted: MiAntenna; antennaUpdated: MiAntenna; @@ -260,12 +266,17 @@ export interface InternalEventTypes { metaUpdated: { before?: MiMeta; after: MiMeta; }; followChannel: { userId: MiUser['id']; channelId: MiChannel['id']; }; unfollowChannel: { userId: MiUser['id']; channelId: MiChannel['id']; }; - updateUserProfile: MiUserProfile; + updateUserProfile: { userId: MiUserProfile['userId'] }; mute: { muterId: MiUser['id']; muteeId: MiUser['id']; }; unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; }; userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; }; + userListMemberUpdated: { userListId: MiUserList['id']; memberId: MiUser['id']; }; userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; }; + userListMemberBulkAdded: { userListIds: MiUserList['id'][]; memberId: MiUser['id']; }; + userListMemberBulkUpdated: { userListIds: MiUserList['id'][]; memberId: MiUser['id']; }; + userListMemberBulkRemoved: { userListIds: MiUserList['id'][]; memberId: MiUser['id']; }; quantumCacheUpdated: { name: string, keys: string[] }; + quantumCacheReset: { name: string }; } type EventTypesToEventPayload = EventUnionFromDictionary>>; @@ -350,6 +361,8 @@ export class GlobalEventService { @Inject(DI.redisForPub) private redisForPub: Redis.Redis, + + private readonly internalEventService: InternalEventService, ) { } @@ -365,81 +378,83 @@ export class GlobalEventService { })); } + /** @deprecated use InternalEventService instead */ @bindThis - public publishInternalEvent(type: K, value?: InternalEventTypes[K]): void { - this.publish('internal', type, typeof value === 'undefined' ? null : value); + public publishInternalEvent(type: K, value: InternalEventTypes[K]): void { + trackPromise(this.internalEventService.emit(type, value)); + } + + /** @deprecated use InternalEventService instead */ + @bindThis + public async publishInternalEventAsync(type: K, value: InternalEventTypes[K]): Promise { + await this.internalEventService.emit(type, value); } @bindThis - public async publishInternalEventAsync(type: K, value?: InternalEventTypes[K]): Promise { - await this.publish('internal', type, typeof value === 'undefined' ? null : value); + public async publishBroadcastStream(type: K, value?: BroadcastTypes[K]): Promise { + await this.publish('broadcast', type, typeof value === 'undefined' ? null : value); } @bindThis - public publishBroadcastStream(type: K, value?: BroadcastTypes[K]): void { - this.publish('broadcast', type, typeof value === 'undefined' ? null : value); + public async publishMainStream(userId: MiUser['id'], type: K, value?: MainEventTypes[K]): Promise { + await this.publish(`mainStream:${userId}`, type, typeof value === 'undefined' ? null : value); } @bindThis - public publishMainStream(userId: MiUser['id'], type: K, value?: MainEventTypes[K]): void { - this.publish(`mainStream:${userId}`, type, typeof value === 'undefined' ? null : value); + public async publishDriveStream(userId: MiUser['id'], type: K, value?: DriveEventTypes[K]): Promise { + await this.publish(`driveStream:${userId}`, type, typeof value === 'undefined' ? null : value); } @bindThis - public publishDriveStream(userId: MiUser['id'], type: K, value?: DriveEventTypes[K]): void { - this.publish(`driveStream:${userId}`, type, typeof value === 'undefined' ? null : value); - } - - @bindThis - public publishNoteStream(noteId: MiNote['id'], type: K, value?: NoteEventTypes[K]): void { - this.publish(`noteStream:${noteId}`, type, { + public async publishNoteStream(noteId: MiNote['id'], type: K, value?: NoteEventTypes[K]): Promise { + await this.publish(`noteStream:${noteId}`, type, { id: noteId, body: value, }); } @bindThis - public publishUserListStream(listId: MiUserList['id'], type: K, value?: UserListEventTypes[K]): void { - this.publish(`userListStream:${listId}`, type, typeof value === 'undefined' ? null : value); + public async publishUserListStream(listId: MiUserList['id'], type: K, value?: UserListEventTypes[K]): Promise { + await this.publish(`userListStream:${listId}`, type, typeof value === 'undefined' ? null : value); } @bindThis - public publishAntennaStream(antennaId: MiAntenna['id'], type: K, value?: AntennaEventTypes[K]): void { - this.publish(`antennaStream:${antennaId}`, type, typeof value === 'undefined' ? null : value); + public async publishAntennaStream(antennaId: MiAntenna['id'], type: K, value?: AntennaEventTypes[K]): Promise { + await this.publish(`antennaStream:${antennaId}`, type, typeof value === 'undefined' ? null : value); } @bindThis - public publishRoleTimelineStream(roleId: MiRole['id'], type: K, value?: RoleTimelineEventTypes[K]): void { - this.publish(`roleTimelineStream:${roleId}`, type, typeof value === 'undefined' ? null : value); + public async publishRoleTimelineStream(roleId: MiRole['id'], type: K, value?: RoleTimelineEventTypes[K]): Promise { + await this.publish(`roleTimelineStream:${roleId}`, type, typeof value === 'undefined' ? null : value); } @bindThis - public publishNotesStream(note: Packed<'Note'>): void { - this.publish('notesStream', null, note); + public async publishNotesStream(note: Packed<'Note'>): Promise { + await this.publish('notesStream', null, note); } @bindThis - public publishAdminStream(userId: MiUser['id'], type: K, value?: AdminEventTypes[K]): void { - this.publish(`adminStream:${userId}`, type, typeof value === 'undefined' ? null : value); + public async publishAdminStream(userId: MiUser['id'], type: K, value?: AdminEventTypes[K]): Promise { + await this.publish(`adminStream:${userId}`, type, typeof value === 'undefined' ? null : value); } @bindThis - public publishChatUserStream(fromUserId: MiUser['id'], toUserId: MiUser['id'], type: K, value?: ChatEventTypes[K]): void { - this.publish(`chatUserStream:${fromUserId}-${toUserId}`, type, typeof value === 'undefined' ? null : value); + public async publishChatUserStream(fromUserId: MiUser['id'], toUserId: MiUser['id'], type: K, value?: ChatEventTypes[K]): Promise { + await this.publish(`chatUserStream:${fromUserId}-${toUserId}`, type, typeof value === 'undefined' ? null : value); } @bindThis - public publishChatRoomStream(toRoomId: MiChatRoom['id'], type: K, value?: ChatEventTypes[K]): void { - this.publish(`chatRoomStream:${toRoomId}`, type, typeof value === 'undefined' ? null : value); + public async publishChatRoomStream(toRoomId: MiChatRoom['id'], type: K, value?: ChatEventTypes[K]): Promise { + await this.publish(`chatRoomStream:${toRoomId}`, type, typeof value === 'undefined' ? null : value); } @bindThis - public publishReversiStream(userId: MiUser['id'], type: K, value?: ReversiEventTypes[K]): void { - this.publish(`reversiStream:${userId}`, type, typeof value === 'undefined' ? null : value); + public async publishReversiStream(userId: MiUser['id'], type: K, value?: ReversiEventTypes[K]): Promise { + await this.publish(`reversiStream:${userId}`, type, typeof value === 'undefined' ? null : value); } @bindThis - public publishReversiGameStream(gameId: MiReversiGame['id'], type: K, value?: ReversiGameEventTypes[K]): void { - this.publish(`reversiGameStream:${gameId}`, type, typeof value === 'undefined' ? null : value); + public async publishReversiGameStream(gameId: MiReversiGame['id'], type: K, value?: ReversiGameEventTypes[K]): Promise { + await this.publish(`reversiGameStream:${gameId}`, type, typeof value === 'undefined' ? null : value); } } diff --git a/packages/backend/src/core/HashtagService.ts b/packages/backend/src/core/HashtagService.ts index 793bbeecb1..0035c4b0d5 100644 --- a/packages/backend/src/core/HashtagService.ts +++ b/packages/backend/src/core/HashtagService.ts @@ -7,14 +7,15 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import type { MiUser } from '@/models/User.js'; +import { isLocalUser, isRemoteUser } from '@/models/User.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import { IdService } from '@/core/IdService.js'; import type { MiHashtag } from '@/models/Hashtag.js'; import type { HashtagsRepository, MiMeta } from '@/models/_.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import { FeaturedService } from '@/core/FeaturedService.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class HashtagService { @@ -28,10 +29,10 @@ export class HashtagService { @Inject(DI.hashtagsRepository) private hashtagsRepository: HashtagsRepository, - private userEntityService: UserEntityService, private featuredService: FeaturedService, private idService: IdService, private utilityService: UtilityService, + private readonly timeService: TimeService, ) { } @@ -78,19 +79,19 @@ export class HashtagService { set.attachedUsersCount = () => '"attachedUsersCount" + 1'; } // 自分が(ローカル内で)初めてこのタグを使ったなら - if (this.userEntityService.isLocalUser(user) && !index.attachedLocalUserIds.some(id => id === user.id)) { + if (isLocalUser(user) && !index.attachedLocalUserIds.some(id => id === user.id)) { set.attachedLocalUserIds = () => `array_append("attachedLocalUserIds", '${user.id}')`; set.attachedLocalUsersCount = () => '"attachedLocalUsersCount" + 1'; } // 自分が(リモートで)初めてこのタグを使ったなら - if (this.userEntityService.isRemoteUser(user) && !index.attachedRemoteUserIds.some(id => id === user.id)) { + if (isRemoteUser(user) && !index.attachedRemoteUserIds.some(id => id === user.id)) { set.attachedRemoteUserIds = () => `array_append("attachedRemoteUserIds", '${user.id}')`; set.attachedRemoteUsersCount = () => '"attachedRemoteUsersCount" + 1'; } } else { set.attachedUserIds = () => `array_remove("attachedUserIds", '${user.id}')`; set.attachedUsersCount = () => '"attachedUsersCount" - 1'; - if (this.userEntityService.isLocalUser(user)) { + if (isLocalUser(user)) { set.attachedLocalUserIds = () => `array_remove("attachedLocalUserIds", '${user.id}')`; set.attachedLocalUsersCount = () => '"attachedLocalUsersCount" - 1'; } else { @@ -105,12 +106,12 @@ export class HashtagService { set.mentionedUsersCount = () => '"mentionedUsersCount" + 1'; } // 自分が(ローカル内で)初めてこのタグを使ったなら - if (this.userEntityService.isLocalUser(user) && !index.mentionedLocalUserIds.some(id => id === user.id)) { + if (isLocalUser(user) && !index.mentionedLocalUserIds.some(id => id === user.id)) { set.mentionedLocalUserIds = () => `array_append("mentionedLocalUserIds", '${user.id}')`; set.mentionedLocalUsersCount = () => '"mentionedLocalUsersCount" + 1'; } // 自分が(リモートで)初めてこのタグを使ったなら - if (this.userEntityService.isRemoteUser(user) && !index.mentionedRemoteUserIds.some(id => id === user.id)) { + if (isRemoteUser(user) && !index.mentionedRemoteUserIds.some(id => id === user.id)) { set.mentionedRemoteUserIds = () => `array_append("mentionedRemoteUserIds", '${user.id}')`; set.mentionedRemoteUsersCount = () => '"mentionedRemoteUsersCount" + 1'; } @@ -133,10 +134,10 @@ export class HashtagService { mentionedRemoteUsersCount: 0, attachedUserIds: [user.id], attachedUsersCount: 1, - attachedLocalUserIds: this.userEntityService.isLocalUser(user) ? [user.id] : [], - attachedLocalUsersCount: this.userEntityService.isLocalUser(user) ? 1 : 0, - attachedRemoteUserIds: this.userEntityService.isRemoteUser(user) ? [user.id] : [], - attachedRemoteUsersCount: this.userEntityService.isRemoteUser(user) ? 1 : 0, + attachedLocalUserIds: isLocalUser(user) ? [user.id] : [], + attachedLocalUsersCount: isLocalUser(user) ? 1 : 0, + attachedRemoteUserIds: isRemoteUser(user) ? [user.id] : [], + attachedRemoteUsersCount: isRemoteUser(user) ? 1 : 0, } as MiHashtag); } else { this.hashtagsRepository.insert({ @@ -144,10 +145,10 @@ export class HashtagService { name: tag, mentionedUserIds: [user.id], mentionedUsersCount: 1, - mentionedLocalUserIds: this.userEntityService.isLocalUser(user) ? [user.id] : [], - mentionedLocalUsersCount: this.userEntityService.isLocalUser(user) ? 1 : 0, - mentionedRemoteUserIds: this.userEntityService.isRemoteUser(user) ? [user.id] : [], - mentionedRemoteUsersCount: this.userEntityService.isRemoteUser(user) ? 1 : 0, + mentionedLocalUserIds: isLocalUser(user) ? [user.id] : [], + mentionedLocalUsersCount: isLocalUser(user) ? 1 : 0, + mentionedRemoteUserIds: isRemoteUser(user) ? [user.id] : [], + mentionedRemoteUsersCount: isRemoteUser(user) ? 1 : 0, attachedUserIds: [], attachedUsersCount: 0, attachedLocalUserIds: [], @@ -166,7 +167,7 @@ export class HashtagService { if (this.utilityService.isKeyWordIncluded(hashtag, this.meta.sensitiveWords)) return; // YYYYMMDDHHmm (10分間隔) - const now = new Date(); + const now = this.timeService.date; now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0); const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`; @@ -197,7 +198,7 @@ export class HashtagService { @bindThis public async getChart(hashtag: string, range: number): Promise { - const now = new Date(); + const now = this.timeService.date; now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0); const redisPipeline = this.redisClient.pipeline(); @@ -217,7 +218,7 @@ export class HashtagService { @bindThis public async getCharts(hashtags: string[], range: number): Promise> { - const now = new Date(); + const now = this.timeService.date; now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0); const redisPipeline = this.redisClient.pipeline(); diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index bd72fefe4f..a77e6a352b 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -19,6 +19,7 @@ import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/val import type { IObject, IObjectWithId } from '@/core/activitypub/type.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { Response } from 'node-fetch'; import type { Socket } from 'node:net'; @@ -72,7 +73,7 @@ export function validateSocketConnect(allowedPrivateNetworks: PrivateNetwork[] | declare module 'node:http' { interface Agent { - createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket; + createConnection(options: net.NetConnectOpts, callback?: (err: Error | null, stream: net.Socket) => void): net.Socket; } } @@ -85,7 +86,7 @@ class HttpRequestServiceAgent extends http.Agent { } @bindThis - public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket { + public createConnection(options: net.NetConnectOpts, callback?: (err: Error | null, stream: net.Socket) => void): net.Socket { const socket = super.createConnection(options, callback) .on('connect', () => { if (process.env.NODE_ENV === 'production') { @@ -105,7 +106,7 @@ class HttpsRequestServiceAgent extends https.Agent { } @bindThis - public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket { + public createConnection(options: net.NetConnectOpts, callback?: (err: Error | null, stream: net.Socket) => void): net.Socket { const socket = super.createConnection(options, callback) .on('connect', () => { if (process.env.NODE_ENV === 'production') { @@ -156,8 +157,10 @@ export class HttpRequestService { constructor( @Inject(DI.config) private config: Config, + private readonly apUtilityService: ApUtilityService, private readonly utilityService: UtilityService, + private readonly timeService: TimeService, ) { const cache = new CacheableLookup({ maxTtl: 3600, // 1hours @@ -343,7 +346,7 @@ export class HttpRequestService { this.utilityService.assertUrl(parsedUrl, allowHttp); const controller = new AbortController(); - setTimeout(() => { + this.timeService.startTimer(() => { controller.abort(); }, timeout); diff --git a/packages/backend/src/core/IdService.ts b/packages/backend/src/core/IdService.ts index 223a8de678..5bc1997a5f 100644 --- a/packages/backend/src/core/IdService.ts +++ b/packages/backend/src/core/IdService.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { ulid } from 'ulid'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; +import { TimeService } from '@/global/TimeService.js'; import { genAid, isSafeAidT, parseAid, parseAidFull } from '@/misc/id/aid.js'; import { genAidx, isSafeAidxT, parseAidx, parseAidxFull } from '@/misc/id/aidx.js'; import { genMeid, isSafeMeidT, parseMeid, parseMeidFull } from '@/misc/id/meid.js'; @@ -22,6 +23,7 @@ export class IdService { constructor( @Inject(DI.config) private config: Config, + private readonly timeService: TimeService, ) { this.method = config.id.toLowerCase(); } @@ -45,7 +47,7 @@ export class IdService { */ @bindThis public gen(time?: number): string { - const t = (!time || (time > Date.now())) ? Date.now() : time; + const t = (!time || (time > this.timeService.now)) ? this.timeService.now : time; switch (this.method) { case 'aid': return genAid(t); diff --git a/packages/backend/src/core/InstanceStatsService.ts b/packages/backend/src/core/InstanceStatsService.ts new file mode 100644 index 0000000000..1ec604a595 --- /dev/null +++ b/packages/backend/src/core/InstanceStatsService.ts @@ -0,0 +1,117 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import { IsNull, MoreThan } from 'typeorm'; +import { CacheManagementService, type ManagedMemorySingleCache } from '@/global/CacheManagementService.js'; +import NotesChart from '@/core/chart/charts/notes.js'; +import UsersChart from '@/core/chart/charts/users.js'; +import { DI } from '@/di-symbols.js'; +import type { UsersRepository } from '@/models/_.js'; +import { bindThis } from '@/decorators.js'; +import { TimeService } from '@/global/TimeService.js'; + +export interface InstanceStats { + /** + * The number of local posts on the instance. + * Updated hourly. + */ + notesTotal: number; + + /** + * The number of local users currently registered on the instance. + * Updated hourly. + */ + usersTotal: number; + + /** + * The number of local users who have been active within the past month. + * Updated daily. + */ + usersActiveMonth: number; + + /** + * The number of local users who have been active within the past 6 months. + * Updated weekly. + */ + usersActiveSixMonths: number; +} + +@Injectable() +export class InstanceStatsService { + private readonly activeSixMonthsCache: ManagedMemorySingleCache; + private readonly activeMonthCache: ManagedMemorySingleCache; + private readonly localUsersCache: ManagedMemorySingleCache; + private readonly localPostsCache: ManagedMemorySingleCache; + + constructor( + @Inject(DI.usersRepository) + private readonly usersRepository: UsersRepository, + + private readonly notesChart: NotesChart, + private readonly usersChart: UsersChart, + private readonly timeService: TimeService, + + cacheManagementService: CacheManagementService, + ) { + this.localPostsCache = cacheManagementService.createMemorySingleCache('localPosts', 1000 * 60 * 60); // 1h + this.localUsersCache = cacheManagementService.createMemorySingleCache('localUsers', 1000 * 60 * 60); // 1h + this.activeMonthCache = cacheManagementService.createMemorySingleCache('activeMonth', 1000 * 60 * 60 * 24); // 1d + this.activeSixMonthsCache = cacheManagementService.createMemorySingleCache('activeSixMonths', 1000 * 60 * 60 * 24 * 7); // 1w + } + + @bindThis + public async fetch(): Promise { + const [notesTotal, usersTotal, usersActiveMonth, usersActiveSixMonths] = await Promise.all([ + this.fetchLocalPosts(), + this.fetchLocalUsers(), + this.fetchActiveMonth(), + this.fetchActiveSixMonths(), + ]); + return { notesTotal, usersTotal, usersActiveMonth, usersActiveSixMonths }; + } + + @bindThis + private async fetchActiveSixMonths(): Promise { + return await this.activeSixMonthsCache.fetch(async () => { + const now = this.timeService.now; + const halfYearAgo = new Date(now - 15552000000); + return await this.usersRepository.countBy({ + host: IsNull(), + isBot: false, + lastActiveDate: MoreThan(halfYearAgo), + }); + }); + } + + @bindThis + private async fetchActiveMonth(): Promise { + return await this.activeMonthCache.fetch(async () => { + const now = this.timeService.now; + const halfYearAgo = new Date(now - 2592000000); + return await this.usersRepository.countBy({ + host: IsNull(), + isBot: false, + lastActiveDate: MoreThan(halfYearAgo), + }); + }); + } + + @bindThis + private async fetchLocalUsers(): Promise { + return await this.localUsersCache.fetch(async () => { + const chart = await this.usersChart.getChart('hour', 1, null); + return chart.local.total[0]; + }); + } + + @bindThis + private async fetchLocalPosts(): Promise { + return await this.localPostsCache.fetch(async () => { + const chart = await this.notesChart.getChart('hour', 1, null); + return chart.local.total[0]; + }); + } +} diff --git a/packages/backend/src/core/LoggerService.ts b/packages/backend/src/core/LoggerService.ts index 25721f0630..40a80ab445 100644 --- a/packages/backend/src/core/LoggerService.ts +++ b/packages/backend/src/core/LoggerService.ts @@ -5,23 +5,25 @@ import { Inject, Injectable } from '@nestjs/common'; import Logger from '@/logger.js'; +import { TimeService } from '@/global/TimeService.js'; +import { EnvService } from '@/global/EnvService.js'; import { bindThis } from '@/decorators.js'; -import type { KEYWORD } from 'color-convert/conversions.js'; -import { envOption } from '@/env.js'; import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; +import type { KEYWORD } from 'color-convert/conversions.js'; @Injectable() export class LoggerService { constructor( - @Inject(DI.config) - private config: Config, + @Inject(DI.console) + protected readonly console: Console, + + protected readonly timeService: TimeService, + protected readonly envService: EnvService, ) { } @bindThis public getLogger(domain: string, color?: KEYWORD | undefined) { - const verbose = this.config.logging?.verbose || envOption.verbose; - return new Logger(domain, color, verbose); + return new Logger(domain, color, this.envService, this.timeService, this.console); } } diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 07f82dc23e..68902ef539 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -12,6 +12,7 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; +import { TimeService, type TimerHandle } from '@/global/TimeService.js'; import { MiInstance } from '@/models/Instance.js'; import { diffArrays } from '@/misc/diff-arrays.js'; import type { MetasRepository } from '@/models/_.js'; @@ -20,7 +21,7 @@ import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() export class MetaService implements OnApplicationShutdown { private cache: MiMeta | undefined; - private intervalId: NodeJS.Timeout; + private intervalId: TimerHandle; constructor( @Inject(DI.redisForSub) @@ -34,16 +35,17 @@ export class MetaService implements OnApplicationShutdown { private featuredService: FeaturedService, private globalEventService: GlobalEventService, + private readonly timeService: TimeService, ) { //this.onMessage = this.onMessage.bind(this); if (process.env.NODE_ENV !== 'test') { - this.intervalId = setInterval(() => { + this.intervalId = this.timeService.startTimer(() => { this.fetch(true).then(meta => { // fetch内でもセットしてるけど仕様変更の可能性もあるため一応 this.cache = meta; }); - }, 1000 * 60 * 5); + }, 1000 * 60 * 5, { repeated: true }); } this.redisForSub.on('message', this.onMessage); @@ -161,7 +163,7 @@ export class MetaService implements OnApplicationShutdown { @bindThis public dispose(): void { - clearInterval(this.intervalId); + this.timeService.stopTimer(this.intervalId); this.redisForSub.off('message', this.onMessage); } diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 0d80bfbdca..49eed40ee1 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -299,7 +299,7 @@ export class MfmService { (note that the `rp` are to be ignored, they only exist for browsers who don't understand ruby) */ - let nonRtNodes = []; + let nonRtNodes: ChildNode[] = []; // scan children, ignore `rp`, split on `rt` for (const child of node.childNodes) { if (isText(child)) { diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index a168126a2f..e7cfbd136b 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -19,6 +19,7 @@ import type { MiApp } from '@/models/App.js'; import { concat } from '@/misc/prelude/array.js'; import { IdService } from '@/core/IdService.js'; import type { MiUser, MiLocalUser, MiRemoteUser } from '@/models/User.js'; +import { isLocalUser, isRemoteUser } from '@/models/User.js'; import type { IPoll } from '@/models/Poll.js'; import { MiPoll } from '@/models/Poll.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; @@ -39,7 +40,6 @@ import { HashtagService } from '@/core/HashtagService.js'; import { AntennaService } from '@/core/AntennaService.js'; import { QueueService } from '@/core/QueueService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; @@ -57,8 +57,8 @@ import { IdentifiableError } from '@/misc/identifiable-error.js'; import { LatestNoteService } from '@/core/LatestNoteService.js'; import { CollapsedQueue } from '@/misc/collapsed-queue.js'; import { CacheService } from '@/core/CacheService.js'; +import { TimeService } from '@/global/TimeService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; -import { isPureRenote } from '@/misc/is-renote.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -199,7 +199,6 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.channelFollowingsRepository) private channelFollowingsRepository: ChannelFollowingsRepository, - private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, private idService: IdService, private globalEventService: GlobalEventService, @@ -225,9 +224,10 @@ export class NoteCreateService implements OnApplicationShutdown { private userBlockingService: UserBlockingService, private cacheService: CacheService, private latestNoteService: LatestNoteService, + private readonly timeService: TimeService, private readonly noteVisibilityService: NoteVisibilityService, ) { - this.updateNotesCountQueue = new CollapsedQueue(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseNotesCount, this.performUpdateNotesCount); + this.updateNotesCountQueue = new CollapsedQueue(this.timeService, process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseNotesCount, this.performUpdateNotesCount); } @bindThis @@ -254,7 +254,7 @@ export class NoteCreateService implements OnApplicationShutdown { data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId }); } - if (data.createdAt == null) data.createdAt = new Date(); + if (data.createdAt == null) data.createdAt = this.timeService.date; if (data.visibility == null) data.visibility = 'public'; if (data.localOnly == null) data.localOnly = false; if (data.channel != null) data.visibility = 'public'; @@ -525,7 +525,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (mentionedUsers.length > 0) { insert.mentions = mentionedUsers.map(u => u.id); const profiles = await this.userProfilesRepository.findBy({ userId: In(insert.mentions) }); - insert.mentionedRemoteUsers = JSON.stringify(mentionedUsers.filter(u => this.userEntityService.isRemoteUser(u)).map(u => { + insert.mentionedRemoteUsers = JSON.stringify(mentionedUsers.filter(u => isRemoteUser(u)).map(u => { const profile = profiles.find(p => p.userId === u.id); const url = profile != null ? profile.url : null; return { @@ -591,7 +591,7 @@ export class NoteCreateService implements OnApplicationShutdown { // Register host if (this.meta.enableStatsForFederatedInstances) { - if (this.userEntityService.isRemoteUser(user)) { + if (isRemoteUser(user)) { this.federatedInstanceService.fetchOrRegister(user.host).then(async i => { if (!this.isRenote(note) || this.isQuote(note)) { this.updateNotesCountQueue.enqueue(i.id, 1); @@ -614,7 +614,7 @@ export class NoteCreateService implements OnApplicationShutdown { // Increment notes count (user) this.incNotesCountOfUser(user); } else { - this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); + this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); } this.pushToTl(note, user); @@ -658,11 +658,11 @@ export class NoteCreateService implements OnApplicationShutdown { } if (data.poll && data.poll.expiresAt) { - const delay = data.poll.expiresAt.getTime() - Date.now(); + const delay = data.poll.expiresAt.getTime() - this.timeService.now; this.queueService.endedPollNotificationQueue.add(note.id, { noteId: note.id, }, { - jobId: `pollEnd:${note.id}`, + jobId: `pollEnd_${note.id}`, delay, removeOnComplete: { age: 3600 * 24 * 7, // keep up to 7 days @@ -676,7 +676,7 @@ export class NoteCreateService implements OnApplicationShutdown { } if (!silent) { - if (this.userEntityService.isLocalUser(user)) this.activeUsersChart.write(user); + if (isLocalUser(user)) this.activeUsersChart.write(user); // Pack the note const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true }); @@ -752,26 +752,26 @@ export class NoteCreateService implements OnApplicationShutdown { nm.notify(); //#region AP deliver - if (!data.localOnly && this.userEntityService.isLocalUser(user)) { + if (!data.localOnly && isLocalUser(user)) { trackTask(async () => { const noteActivity = await this.apRendererService.renderNoteOrRenoteActivity(note, user, { renote: data.renote }); const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity); // メンションされたリモートユーザーに配送 - for (const u of mentionedUsers.filter(u => this.userEntityService.isRemoteUser(u))) { + for (const u of mentionedUsers.filter(u => isRemoteUser(u))) { dm.addDirectRecipe(u as MiRemoteUser); } // 投稿がリプライかつ投稿者がローカルユーザーかつリプライ先の投稿の投稿者がリモートユーザーなら配送 if (data.reply && data.reply.userHost !== null) { const u = await this.usersRepository.findOneBy({ id: data.reply.userId }); - if (u && this.userEntityService.isRemoteUser(u)) dm.addDirectRecipe(u); + if (u && isRemoteUser(u)) dm.addDirectRecipe(u); } // 投稿がRenoteかつ投稿者がローカルユーザーかつRenote元の投稿の投稿者がリモートユーザーなら配送 if (data.renote && data.renote.userHost !== null) { const u = await this.usersRepository.findOneBy({ id: data.renote.userId }); - if (u && this.userEntityService.isRemoteUser(u)) dm.addDirectRecipe(u); + if (u && isRemoteUser(u)) dm.addDirectRecipe(u); } // フォロワーに配送 @@ -792,7 +792,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (data.channel) { this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); this.channelsRepository.update(data.channel.id, { - lastNotedAt: new Date(), + lastNotedAt: this.timeService.date, }); this.notesRepository.countBy({ @@ -814,27 +814,20 @@ export class NoteCreateService implements OnApplicationShutdown { if (!user.noindex) this.index(note); } - @bindThis - public isPureRenote(note: Option): note is PureRenoteOption { - return this.isRenote(note) && !this.isQuote(note); - } + /** + * @deprecated Use the exported function instead + */ + readonly isPureRenote = isPureRenote; - @bindThis - private isRenote(note: Option): note is Option & { renote: MiNote } { - return note.renote != null; - } + /** + * @deprecated Use the exported function instead + */ + readonly isRenote = isRenote; - @bindThis - private isQuote(note: Option & { renote: MiNote }): note is Option & { renote: MiNote } & ( - { text: string } | { cw: string } | { reply: MiNote } | { poll: IPoll } | { files: MiDriveFile[] } - ) { - // NOTE: SYNC WITH misc/is-quote.ts - return note.text != null || - note.reply != null || - note.cw != null || - note.poll != null || - (note.files != null && note.files.length > 0); - } + /** + * @deprecated Use the exported function instead + */ + readonly isQuote = isQuote; @bindThis private async incRenoteCount(renote: MiNote, user: MiUser) { @@ -846,7 +839,7 @@ export class NoteCreateService implements OnApplicationShutdown { .execute(); // 30%の確率、3日以内に投稿されたノートの場合ハイライト用ランキング更新 - if (user.isExplorable && Math.random() < 0.3 && (Date.now() - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) { + if (user.isExplorable && Math.random() < 0.3 && (this.timeService.now - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) { const policies = await this.roleService.getUserPolicies(user); if (policies.canTrend) { if (renote.channelId != null) { @@ -874,7 +867,7 @@ export class NoteCreateService implements OnApplicationShutdown { ]); // Only create mention events for local users, and users for whom the note is visible - for (const u of mentionedUsers.filter(u => (note.visibility !== 'specified' || note.visibleUserIds.some(x => x === u.id)) && this.userEntityService.isLocalUser(u))) { + for (const u of mentionedUsers.filter(u => (note.visibility !== 'specified' || note.visibleUserIds.some(x => x === u.id)) && isLocalUser(u))) { const threadId = note.threadId ?? note.id; const isThreadMuted = threadMutings.get(u.id)?.has(threadId); @@ -913,7 +906,7 @@ export class NoteCreateService implements OnApplicationShutdown { private incNotesCountOfUser(user: { id: MiUser['id']; }) { this.usersRepository.createQueryBuilder().update() .set({ - updatedAt: new Date(), + updatedAt: this.timeService.date, notesCount: () => '"notesCount" + 1', }) .where('id = :id', { id: user.id }) @@ -966,12 +959,7 @@ export class NoteCreateService implements OnApplicationShutdown { // eslint-disable-next-line prefer-const let [followings, userListMemberships] = await Promise.all([ this.cacheService.getNonHibernatedFollowers(user.id), - this.userListMembershipsRepository.find({ - where: { - userId: user.id, - }, - select: ['userListId', 'userListUserId', 'withReplies'], - }), + this.cacheService.userListMembershipsCache.fetch(user.id).then(ms => ms.values().toArray()), ]); if (note.visibility === 'followers') { @@ -1077,7 +1065,7 @@ export class NoteCreateService implements OnApplicationShutdown { const hibernatedUsers = await this.usersRepository.find({ where: { id: In(samples.map(x => x.followerId)), - lastActiveDate: LessThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 50))), + lastActiveDate: LessThan(new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 50))), }, select: ['id'], }); @@ -1162,3 +1150,22 @@ export class NoteCreateService implements OnApplicationShutdown { } } } + +export function isPureRenote(note: Option): note is PureRenoteOption { + return isRenote(note) && !isQuote(note); +} + +export function isRenote(note: Option): note is Option & { renote: MiNote } { + return note.renote != null; +} + +export function isQuote(note: Option & { renote: MiNote }): note is Option & { renote: MiNote } & ( + { text: string } | { cw: string } | { reply: MiNote } | { poll: IPoll } | { files: MiDriveFile[] } +) { + // NOTE: SYNC WITH misc/is-quote.ts + return note.text != null || + note.reply != null || + note.cw != null || + note.poll != null || + (note.files != null && note.files.length > 0); +} diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index 9ce8cb6731..95bdf85308 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -6,6 +6,7 @@ import { Brackets, In, IsNull, Not } from 'typeorm'; import { Injectable, Inject } from '@nestjs/common'; import type { MiUser, MiLocalUser, MiRemoteUser } from '@/models/User.js'; +import { isLocalUser, isRemoteUser } from '@/models/User.js'; import { MiNote, IMentionedRemoteUsers } from '@/models/Note.js'; import type { InstancesRepository, MiMeta, NotesRepository, UsersRepository } from '@/models/_.js'; import { RelayService } from '@/core/RelayService.js'; @@ -18,15 +19,15 @@ import InstanceChart from '@/core/chart/charts/instance.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import { SearchService } from '@/core/SearchService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; import { LatestNoteService } from '@/core/LatestNoteService.js'; import { ApLogService } from '@/core/ApLogService.js'; -import Logger from '@/logger.js'; -import { LoggerService } from './LoggerService.js'; +import type Logger from '@/logger.js'; +import { TimeService } from '@/global/TimeService.js'; +import { LoggerService } from '@/core/LoggerService.js'; @Injectable() export class NoteDeleteService { @@ -48,7 +49,6 @@ export class NoteDeleteService { @Inject(DI.instancesRepository) private instancesRepository: InstancesRepository, - private userEntityService: UserEntityService, private globalEventService: GlobalEventService, private relayService: RelayService, private federatedInstanceService: FederatedInstanceService, @@ -61,6 +61,8 @@ export class NoteDeleteService { private instanceChart: InstanceChart, private latestNoteService: LatestNoteService, private readonly apLogService: ApLogService, + private readonly timeService: TimeService, + loggerService: LoggerService, ) { this.logger = loggerService.getLogger('note-delete-service'); @@ -72,7 +74,7 @@ export class NoteDeleteService { * @param note 投稿 */ async delete(user: { id: MiUser['id']; uri: MiUser['uri']; host: MiUser['host']; isBot: MiUser['isBot']; }, note: MiNote, quiet = false, deleter?: MiUser) { - const deletedAt = new Date(); + const deletedAt = this.timeService.date; const cascadingNotes = await this.findCascadingNotes(note); if (note.replyId) { @@ -92,7 +94,7 @@ export class NoteDeleteService { }); //#region ローカルの投稿なら削除アクティビティを配送 - if (this.userEntityService.isLocalUser(user) && !note.localOnly) { + if (isLocalUser(user) && !note.localOnly) { let renote: MiNote | null = null; // if deleted note is renote @@ -113,7 +115,7 @@ export class NoteDeleteService { const federatedLocalCascadingNotes = (cascadingNotes).filter(note => !note.localOnly && note.userHost == null); // filter out local-only notes for (const cascadingNote of federatedLocalCascadingNotes) { if (!cascadingNote.user) continue; - if (!this.userEntityService.isLocalUser(cascadingNote.user)) continue; + if (!isLocalUser(cascadingNote.user)) continue; const content = this.apRendererService.addContext(this.apRendererService.renderDelete(this.apRendererService.renderTombstone(`${this.config.url}/notes/${cascadingNote.id}`), cascadingNote.user)); this.deliverToConcerned(cascadingNote.user, cascadingNote, content); } @@ -128,11 +130,11 @@ export class NoteDeleteService { // Decrement notes count (user) this.decNotesCountOfUser(user); } else { - this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); + this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); } if (this.meta.enableStatsForFederatedInstances) { - if (this.userEntityService.isRemoteUser(user)) { + if (isRemoteUser(user)) { this.federatedInstanceService.fetchOrRegister(user.host).then(async i => { if (note.renoteId && note.text || !note.renoteId) { this.instancesRepository.decrement({ id: i.id }, 'notesCount', 1); @@ -180,7 +182,7 @@ export class NoteDeleteService { private decNotesCountOfUser(user: { id: MiUser['id']; }) { this.usersRepository.createQueryBuilder().update() .set({ - updatedAt: new Date(), + updatedAt: this.timeService.date, notesCount: () => '"notesCount" - 1', }) .where('id = :id', { id: user.id }) diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index af9538dc50..17daf386d6 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -20,6 +20,7 @@ import type { MiApp } from '@/models/App.js'; import { concat } from '@/misc/prelude/array.js'; import { IdService } from '@/core/IdService.js'; import type { MiUser, MiLocalUser, MiRemoteUser } from '@/models/User.js'; +import { isRemoteUser, isLocalUser } from '@/models/User.js'; import { MiPoll, type IPoll } from '@/models/Poll.js'; import type { MiChannel } from '@/models/Channel.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; @@ -34,7 +35,6 @@ import { NotificationService } from '@/core/NotificationService.js'; import { UserWebhookService } from '@/core/UserWebhookService.js'; import { QueueService } from '@/core/QueueService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js'; @@ -52,6 +52,7 @@ import { IdentifiableError } from '@/misc/identifiable-error.js'; import { LatestNoteService } from '@/core/LatestNoteService.js'; import { CollapsedQueue } from '@/misc/collapsed-queue.js'; import { NoteCreateService } from '@/core/NoteCreateService.js'; +import { TimeService } from '@/global/TimeService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import { isPureRenote } from '@/misc/is-renote.js'; @@ -200,7 +201,6 @@ export class NoteEditService implements OnApplicationShutdown { @Inject(DI.pollsRepository) private pollsRepository: PollsRepository, - private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, private idService: IdService, private globalEventService: GlobalEventService, @@ -222,9 +222,10 @@ export class NoteEditService implements OnApplicationShutdown { private cacheService: CacheService, private latestNoteService: LatestNoteService, private noteCreateService: NoteCreateService, + private readonly timeService: TimeService, private readonly noteVisibilityService: NoteVisibilityService, ) { - this.updateNotesCountQueue = new CollapsedQueue(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseNotesCount, this.performUpdateNotesCount); + this.updateNotesCountQueue = new CollapsedQueue(this.timeService, process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseNotesCount, this.performUpdateNotesCount); } @bindThis @@ -273,7 +274,7 @@ export class NoteEditService implements OnApplicationShutdown { data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId }); } - if (data.updatedAt == null) data.updatedAt = new Date(); + if (data.updatedAt == null) data.updatedAt = this.timeService.date; if (data.visibility == null) data.visibility = 'public'; if (data.localOnly == null) data.localOnly = false; if (data.channel != null) data.visibility = 'public'; @@ -494,12 +495,12 @@ export class NoteEditService implements OnApplicationShutdown { cw: update.cw || undefined, fileIds: undefined, oldDate: exists ? oldnote.updatedAt as Date : this.idService.parse(oldnote.id).date, - updatedAt: new Date(), + updatedAt: this.timeService.date, }); const note = new MiNote({ id: oldnote.id, - updatedAt: data.updatedAt ? data.updatedAt : new Date(), + updatedAt: data.updatedAt ? data.updatedAt : this.timeService.date, fileIds: data.files ? data.files.map(file => file.id) : [], replyId: oldnote.replyId, renoteId: data.renote ? data.renote.id : null, @@ -545,7 +546,7 @@ export class NoteEditService implements OnApplicationShutdown { if (mentionedUsers.length > 0) { note.mentions = mentionedUsers.map(u => u.id); const profiles = await this.userProfilesRepository.findBy({ userId: In(note.mentions) }); - note.mentionedRemoteUsers = JSON.stringify(mentionedUsers.filter(u => this.userEntityService.isRemoteUser(u)).map(u => { + note.mentionedRemoteUsers = JSON.stringify(mentionedUsers.filter(u => isRemoteUser(u)).map(u => { const profile = profiles.find(p => p.userId === u.id); const url = profile != null ? profile.url : null; return { @@ -608,7 +609,7 @@ export class NoteEditService implements OnApplicationShutdown { }, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) { // Register host if (this.meta.enableStatsForFederatedInstances) { - if (this.userEntityService.isRemoteUser(user)) { + if (isRemoteUser(user)) { this.federatedInstanceService.fetchOrRegister(user.host).then(async i => { if (note.renote && note.text || !note.renote) { this.updateNotesCountQueue.enqueue(i.id, 1); @@ -620,25 +621,25 @@ export class NoteEditService implements OnApplicationShutdown { } } - this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); + this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); // ハッシュタグ更新 this.pushToTl(note, user); if (data.poll && data.poll.expiresAt) { - const delay = data.poll.expiresAt.getTime() - Date.now(); + const delay = data.poll.expiresAt.getTime() - this.timeService.now; this.queueService.endedPollNotificationQueue.remove(`pollEnd:${note.id}`); this.queueService.endedPollNotificationQueue.add(note.id, { noteId: note.id, }, { - jobId: `pollEnd:${note.id}`, + jobId: `pollEnd_${note.id}`, delay, removeOnComplete: true, }); } if (!silent) { - if (this.userEntityService.isLocalUser(user)) this.activeUsersChart.write(user); + if (isLocalUser(user)) this.activeUsersChart.write(user); // Pack the note const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true }); @@ -680,26 +681,26 @@ export class NoteEditService implements OnApplicationShutdown { nm.notify(); //#region AP deliver - if (!data.localOnly && this.userEntityService.isLocalUser(user)) { + if (!data.localOnly && isLocalUser(user)) { trackTask(async () => { const noteActivity = await this.apRendererService.renderNoteOrRenoteActivity(note, user, { renote: data.renote }); const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity); // メンションされたリモートユーザーに配送 - for (const u of mentionedUsers.filter(u => this.userEntityService.isRemoteUser(u))) { + for (const u of mentionedUsers.filter(u => isRemoteUser(u))) { dm.addDirectRecipe(u as MiRemoteUser); } // 投稿がリプライかつ投稿者がローカルユーザーかつリプライ先の投稿の投稿者がリモートユーザーなら配送 if (data.reply && data.reply.userHost !== null) { const u = await this.usersRepository.findOneBy({ id: data.reply.userId }); - if (u && this.userEntityService.isRemoteUser(u)) dm.addDirectRecipe(u); + if (u && isRemoteUser(u)) dm.addDirectRecipe(u); } // 投稿がRenoteかつ投稿者がローカルユーザーかつRenote元の投稿の投稿者がリモートユーザーなら配送 if (this.isRenote(data) && data.renote.userHost !== null) { const u = await this.usersRepository.findOneBy({ id: data.renote.userId }); - if (u && this.userEntityService.isRemoteUser(u)) dm.addDirectRecipe(u); + if (u && isRemoteUser(u)) dm.addDirectRecipe(u); } // フォロワーに配送 @@ -738,7 +739,7 @@ export class NoteEditService implements OnApplicationShutdown { if (data.channel) { this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); this.channelsRepository.update(data.channel.id, { - lastNotedAt: new Date(), + lastNotedAt: this.timeService.date, }); this.notesRepository.countBy({ @@ -830,12 +831,7 @@ export class NoteEditService implements OnApplicationShutdown { // eslint-disable-next-line prefer-const let [followings, userListMemberships] = await Promise.all([ this.cacheService.getNonHibernatedFollowers(user.id), - this.userListMembershipsRepository.find({ - where: { - userId: user.id, - }, - select: ['userListId', 'userListUserId', 'withReplies'], - }), + this.cacheService.userListMembershipsCache.fetch(user.id).then(ms => ms.values().toArray()), ]); if (note.visibility === 'followers') { @@ -941,7 +937,7 @@ export class NoteEditService implements OnApplicationShutdown { const hibernatedUsers = await this.usersRepository.find({ where: { id: In(samples.map(x => x.followerId)), - lastActiveDate: LessThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 50))), + lastActiveDate: LessThan(new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 50))), }, select: ['id'], }); diff --git a/packages/backend/src/core/NoteVisibilityService.ts b/packages/backend/src/core/NoteVisibilityService.ts index 0285847cf5..a0cc3a85aa 100644 --- a/packages/backend/src/core/NoteVisibilityService.ts +++ b/packages/backend/src/core/NoteVisibilityService.ts @@ -4,15 +4,19 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { CacheService } from '@/core/CacheService.js'; import type { MiNote } from '@/models/Note.js'; import type { MiUser } from '@/models/User.js'; -import { bindThis } from '@/decorators.js'; +import type { MiFollowing } from '@/models/Following.js'; +import type { MiInstance } from '@/models/Instance.js'; +import type { MiUserListMembership } from '@/models/UserListMembership.js'; +import type { NotesRepository } from '@/models/_.js'; import type { Packed } from '@/misc/json-schema.js'; -import { IdService } from '@/core/IdService.js'; -import { awaitAll } from '@/misc/prelude/await-all.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; -import type { MiFollowing, MiInstance, NotesRepository } from '@/models/_.js'; +import { CacheService } from '@/core/CacheService.js'; +import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; +import { bindThis } from '@/decorators.js'; +import { awaitAll } from '@/misc/prelude/await-all.js'; import { DI } from '@/di-symbols.js'; /** @@ -50,6 +54,12 @@ export interface NoteVisibilityFilters { * If false (default), then silence is enforced for all notes. */ includeSilencedAuthor?: boolean; + + /** + * Set to an ID to apply visibility from the context of a specific user list. + * Membership and "with replies" settings will be adopted from this list. + */ + listContext?: string | null; } @Injectable() @@ -61,6 +71,7 @@ export class NoteVisibilityService { private readonly cacheService: CacheService, private readonly idService: IdService, private readonly federatedInstanceService: FederatedInstanceService, + private readonly timeService: TimeService, ) {} @bindThis @@ -151,7 +162,7 @@ export class NoteVisibilityService { } @bindThis - public async populateData(user: PopulatedMe, hint?: Partial): Promise { + public async populateData(user: PopulatedMe, hint?: Partial, filters?: NoteVisibilityFilters): Promise { // noinspection ES6MissingAwait const [ userBlockers, @@ -161,6 +172,7 @@ export class NoteVisibilityService { userMutedUsers, userMutedUserRenotes, userMutedInstances, + userListMemberships, ] = await Promise.all([ user ? (hint?.userBlockers ?? this.cacheService.userBlockedCache.fetch(user.id)) : null, user ? (hint?.userFollowings ?? this.cacheService.userFollowingsCache.fetch(user.id)) : null, @@ -169,6 +181,7 @@ export class NoteVisibilityService { user ? (hint?.userMutedUsers ?? this.cacheService.userMutingsCache.fetch(user.id)) : null, user ? (hint?.userMutedUserRenotes ?? this.cacheService.renoteMutingsCache.fetch(user.id)) : null, user ? (hint?.userMutedInstances ?? this.cacheService.userProfileCache.fetch(user.id).then(p => new Set(p.mutedInstances))) : null, + filters?.listContext ? (hint?.userListMemberships ?? this.cacheService.listUserMembershipsCache.fetch(filters.listContext)) : null, ]); return { @@ -179,6 +192,7 @@ export class NoteVisibilityService { userMutedUsers, userMutedUserRenotes, userMutedInstances, + userListMemberships, }; } @@ -281,7 +295,7 @@ export class NoteVisibilityService { const createdAt = new Date(note.createdAt).valueOf(); // I don't understand this logic, but I tried to break it out for readability - const followersOnlyOpt1 = followersOnlyBefore <= 0 && (Date.now() - createdAt > 0 - followersOnlyBefore); + const followersOnlyOpt1 = followersOnlyBefore <= 0 && (this.timeService.now - createdAt > 0 - followersOnlyBefore); const followersOnlyOpt2 = followersOnlyBefore > 0 && (createdAt < followersOnlyBefore); if (followersOnlyOpt1 || followersOnlyOpt2) { note.visibility = 'followers'; @@ -311,7 +325,7 @@ export class NoteVisibilityService { const createdAt = note.createdAt.valueOf(); // I don't understand this logic, but I tried to break it out for readability - const hiddenOpt1 = hiddenBefore <= 0 && (Date.now() - createdAt > 0 - hiddenBefore); + const hiddenOpt1 = hiddenBefore <= 0 && (this.timeService.now - createdAt > 0 - hiddenBefore); const hiddenOpt2 = hiddenBefore > 0 && (createdAt < hiddenBefore); if (hiddenOpt1 || hiddenOpt2) return true; } @@ -396,6 +410,9 @@ export class NoteVisibilityService { // Don't silence if we follow w/ replies if (user && data.userFollowings?.get(user.id)?.withReplies) return false; + // Don't silence if we're viewing in a list with replies + if (data.userListMemberships?.get(note.userId)?.withReplies) return false; + // Silence otherwise return true; } @@ -409,6 +426,9 @@ export interface NoteVisibilityData extends NotePopulationData { userMutedUsers: Set | null; userMutedUserRenotes: Set | null; userMutedInstances: Set | null; + + // userId => membership (already scoped to listContext) + userListMemberships: Map | null; } export interface NotePopulationData { diff --git a/packages/backend/src/core/NotificationService.ts b/packages/backend/src/core/NotificationService.ts index 2ce7bdb5a9..9180dfa418 100644 --- a/packages/backend/src/core/NotificationService.ts +++ b/packages/backend/src/core/NotificationService.ts @@ -22,6 +22,7 @@ import type { Config } from '@/config.js'; import { UserListService } from '@/core/UserListService.js'; import { FilterUnionByProperty, groupedNotificationTypes, obsoleteNotificationTypes } from '@/types.js'; import { trackPromise } from '@/misc/promise-tracker.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class NotificationService implements OnApplicationShutdown { @@ -43,6 +44,7 @@ export class NotificationService implements OnApplicationShutdown { private pushNotificationService: PushNotificationService, private cacheService: CacheService, private userListService: UserListService, + private readonly timeService: TimeService, ) { } @@ -93,7 +95,10 @@ export class NotificationService implements OnApplicationShutdown { data: Omit, 'type' | 'id' | 'createdAt' | 'notifierId'>, notifierId?: MiUser['id'] | null, ): Promise { - const profile = await this.cacheService.userProfileCache.fetch(notifieeId); + const [profile, notifiee] = await Promise.all([ + this.cacheService.userProfileCache.fetch(notifieeId), + this.cacheService.findUserById(notifieeId), + ]); // 古いMisskeyバージョンのキャッシュが残っている可能性がある // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition @@ -139,14 +144,14 @@ export class NotificationService implements OnApplicationShutdown { return null; } } else if (recieveConfig?.type === 'list') { - const isMember = await this.userListService.membersCache.fetch(recieveConfig.userListId).then(members => members.has(notifierId)); + const isMember = await this.cacheService.listUserMembershipsCache.fetch(recieveConfig.userListId).then(members => members.has(notifierId)); if (!isMember) { return null; } } } - const createdAt = new Date(); + const createdAt = this.timeService.date; let notification: FilterUnionByProperty; let redisId: string; @@ -177,7 +182,7 @@ export class NotificationService implements OnApplicationShutdown { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition } while (true); - const packed = await this.notificationEntityService.pack(notification, notifieeId, {}); + const packed = await this.notificationEntityService.pack(notification, notifiee, {}); if (packed == null) return null; @@ -187,15 +192,15 @@ export class NotificationService implements OnApplicationShutdown { // 2秒経っても(今回作成した)通知が既読にならなかったら「未読の通知がありますよ」イベントを発行する // テスト通知の場合は即時発行 const interval = notification.type === 'test' ? 0 : 2000; - setTimeout(interval, 'unread notification', { signal: this.#shutdownController.signal }).then(async () => { + this.timeService.startPromiseTimer(interval, 'unread notification', { signal: this.#shutdownController.signal }).then(async () => { const latestReadNotificationId = await this.redisClient.get(`latestReadNotification:${notifieeId}`); if (latestReadNotificationId && (latestReadNotificationId >= redisId)) return; this.globalEventService.publishMainStream(notifieeId, 'unreadNotification', packed); this.pushNotificationService.pushNotification(notifieeId, 'notification', packed); - if (type === 'follow') this.emailNotificationFollow(notifieeId, await this.usersRepository.findOneByOrFail({ id: notifierId! })); - if (type === 'receiveFollowRequest') this.emailNotificationReceiveFollowRequest(notifieeId, await this.usersRepository.findOneByOrFail({ id: notifierId! })); + if (type === 'follow') this.emailNotificationFollow(notifieeId, await this.cacheService.findUserById(notifierId!)); + if (type === 'receiveFollowRequest') this.emailNotificationReceiveFollowRequest(notifieeId, await this.cacheService.findUserById(notifierId!)); }, () => { /* aborted, ignore it */ }); return notification; diff --git a/packages/backend/src/core/PollService.ts b/packages/backend/src/core/PollService.ts index 33262a4804..e93cc7ba4c 100644 --- a/packages/backend/src/core/PollService.ts +++ b/packages/backend/src/core/PollService.ts @@ -11,10 +11,10 @@ import { RelayService } from '@/core/RelayService.js'; import { IdService } from '@/core/IdService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; import { bindThis } from '@/decorators.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; +import { isLocalUser } from '@/models/User.js'; @Injectable() export class PollService { @@ -31,7 +31,6 @@ export class PollService { @Inject(DI.pollVotesRepository) private pollVotesRepository: PollVotesRepository, - private userEntityService: UserEntityService, private idService: IdService, private relayService: RelayService, private globalEventService: GlobalEventService, @@ -96,7 +95,7 @@ export class PollService { const user = await this.usersRepository.findOneBy({ id: note.userId }); if (user == null) throw new Error('note not found'); - if (this.userEntityService.isLocalUser(user)) { + if (isLocalUser(user)) { const content = this.apRendererService.addContext(this.apRendererService.renderUpdate(await this.apRendererService.renderNote(note, user, false), user)); await this.apDeliverManagerService.deliverToFollowers(user, content); await this.relayService.deliverToRelays(user, content); diff --git a/packages/backend/src/core/PushNotificationService.ts b/packages/backend/src/core/PushNotificationService.ts index e3f10d4504..2697812f73 100644 --- a/packages/backend/src/core/PushNotificationService.ts +++ b/packages/backend/src/core/PushNotificationService.ts @@ -12,8 +12,8 @@ import type { Packed } from '@/misc/json-schema.js'; import { getNoteSummary } from '@/misc/get-note-summary.js'; import type { MiMeta, MiSwSubscription, SwSubscriptionsRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; -import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; +import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js'; +import { TimeService } from '@/global/TimeService.js'; // Defined also packages/sw/types.ts#L13 type PushNotificationsTypes = { @@ -48,8 +48,8 @@ function truncateBody(type: T, body: Pus } @Injectable() -export class PushNotificationService implements OnApplicationShutdown { - private subscriptionsCache: QuantumKVCache; +export class PushNotificationService { + private readonly subscriptionsCache: ManagedQuantumKVCache; constructor( @Inject(DI.config) @@ -63,11 +63,16 @@ export class PushNotificationService implements OnApplicationShutdown { @Inject(DI.swSubscriptionsRepository) private swSubscriptionsRepository: SwSubscriptionsRepository, - private readonly internalEventService: InternalEventService, + + private readonly timeService: TimeService, + + cacheManagementService: CacheManagementService, ) { - this.subscriptionsCache = new QuantumKVCache(this.internalEventService, 'userSwSubscriptions', { + this.subscriptionsCache = cacheManagementService.createQuantumKVCache('userSwSubscriptions', { lifetime: 1000 * 60 * 60 * 1, // 1h - fetcher: (key) => this.swSubscriptionsRepository.findBy({ userId: key }), + fetcher: async userId => await this.swSubscriptionsRepository.findBy({ userId }), + // optionalFetcher not needed + // bulkFetcher not needed }); } @@ -99,7 +104,7 @@ export class PushNotificationService implements OnApplicationShutdown { type, body: (type === 'notification' || type === 'unreadAntennaNote') ? truncateBody(type, body) : body, userId, - dateTime: Date.now(), + dateTime: this.timeService.now, }), { proxy: this.config.proxy, }).catch((err: any) => { @@ -125,14 +130,4 @@ export class PushNotificationService implements OnApplicationShutdown { public async refreshCache(userId: string): Promise { await this.subscriptionsCache.refresh(userId); } - - @bindThis - public dispose(): void { - this.subscriptionsCache.dispose(); - } - - @bindThis - public onApplicationShutdown(signal?: string | undefined): void { - this.dispose(); - } } diff --git a/packages/backend/src/core/QueueModule.ts b/packages/backend/src/core/QueueModule.ts index a48ffaab43..2f594394a6 100644 --- a/packages/backend/src/core/QueueModule.ts +++ b/packages/backend/src/core/QueueModule.ts @@ -10,6 +10,8 @@ import type { Config } from '@/config.js'; import { baseQueueOptions, QUEUE } from '@/queue/const.js'; import { allSettled } from '@/misc/promise-tracker.js'; import Logger from '@/logger.js'; +import { bindThis } from '@/decorators.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { DeliverJobData, EndedPollNotificationJobData, @@ -142,7 +144,7 @@ export class QueueModule implements OnApplicationShutdown { await allSettled(); // And then close all queues this.logger.info('Closing BullMQ queues...'); - await Promise.all([ + await Promise.allSettled([ this.systemQueue.close(), this.endedPollNotificationQueue.close(), this.deliverQueue.close(), @@ -153,10 +155,17 @@ export class QueueModule implements OnApplicationShutdown { this.userWebhookDeliverQueue.close(), this.systemWebhookDeliverQueue.close(), this.scheduleNotePostQueue.close(), - ]); + ]).then(res => { + for (const result of res) { + if (result.status === 'rejected') { + this.logger.error(`Error closing queue: ${renderInlineError(result.reason)}`); + } + } + }); this.logger.info('Queue module disposed.'); } + @bindThis async onApplicationShutdown(signal: string): Promise { await this.dispose(); } diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 6cc6008567..5020614676 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -4,7 +4,7 @@ */ import { randomUUID } from 'node:crypto'; -import { Inject, Injectable } from '@nestjs/common'; +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; import { MetricsTime, type JobType } from 'bullmq'; import { parse as parseRedisInfo } from 'redis-info'; import type { IActivity } from '@/core/activitypub/type.js'; @@ -16,8 +16,9 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; import { ApRequestCreator } from '@/core/activitypub/ApRequestService.js'; -import { type SystemWebhookPayload } from '@/core/SystemWebhookService.js'; -import { MiNote } from '@/models/Note.js'; +import { TimeService } from '@/global/TimeService.js'; +import type { SystemWebhookPayload } from '@/core/SystemWebhookService.js'; +import type { MiNote } from '@/models/Note.js'; import { type UserWebhookPayload } from './UserWebhookService.js'; import type { DbJobData, @@ -56,7 +57,7 @@ export const QUEUE_TYPES = [ ] as const; @Injectable() -export class QueueService { +export class QueueService implements OnModuleInit { constructor( @Inject(DI.config) private config: Config, @@ -71,57 +72,62 @@ export class QueueService { @Inject('queue:userWebhookDeliver') public userWebhookDeliverQueue: UserWebhookDeliverQueue, @Inject('queue:systemWebhookDeliver') public systemWebhookDeliverQueue: SystemWebhookDeliverQueue, @Inject('queue:scheduleNotePost') public ScheduleNotePostQueue: ScheduleNotePostQueue, - ) { - this.systemQueue.add('tickCharts', { + + private readonly timeService: TimeService, + ) {} + + @bindThis + public async onModuleInit() { + await this.systemQueue.add('tickCharts', { }, { repeat: { pattern: '55 * * * *' }, removeOnComplete: 10, removeOnFail: 30, }); - this.systemQueue.add('resyncCharts', { + await this.systemQueue.add('resyncCharts', { }, { repeat: { pattern: '0 0 * * *' }, removeOnComplete: 10, removeOnFail: 30, }); - this.systemQueue.add('cleanCharts', { + await this.systemQueue.add('cleanCharts', { }, { repeat: { pattern: '0 0 * * *' }, removeOnComplete: 10, removeOnFail: 30, }); - this.systemQueue.add('aggregateRetention', { + await this.systemQueue.add('aggregateRetention', { }, { repeat: { pattern: '0 0 * * *' }, removeOnComplete: 10, removeOnFail: 30, }); - this.systemQueue.add('clean', { + await this.systemQueue.add('clean', { }, { repeat: { pattern: '0 0 * * *' }, removeOnComplete: 10, removeOnFail: 30, }); - this.systemQueue.add('checkExpiredMutings', { + await this.systemQueue.add('checkExpiredMutings', { }, { repeat: { pattern: '*/5 * * * *' }, removeOnComplete: 10, removeOnFail: 30, }); - this.systemQueue.add('bakeBufferedReactions', { + await this.systemQueue.add('bakeBufferedReactions', { }, { repeat: { pattern: '0 0 * * *' }, removeOnComplete: 10, removeOnFail: 30, }); - this.systemQueue.add('checkModeratorsActivity', { + await this.systemQueue.add('checkModeratorsActivity', { }, { // 毎時30分に起動 repeat: { pattern: '30 * * * *' }, @@ -788,7 +794,7 @@ export class QueueService { userId: webhook.userId, to: webhook.url, secret: webhook.secret, - createdAt: Date.now(), + createdAt: this.timeService.now, eventId: randomUUID(), }; @@ -825,7 +831,7 @@ export class QueueService { webhookId: webhook.id, to: webhook.url, secret: webhook.secret, - createdAt: Date.now(), + createdAt: this.timeService.now, eventId: randomUUID(), }; @@ -890,7 +896,7 @@ export class QueueService { @bindThis public async queueRetryJob(queueType: typeof QUEUE_TYPES[number], jobId: string) { const queue = this.getQueue(queueType); - const job: Bull.Job | null = await queue.getJob(jobId); + const job: Bull.Job | undefined = await queue.getJob(jobId); if (job) { if (job.finishedOn != null) { await job.retry(); @@ -903,7 +909,7 @@ export class QueueService { @bindThis public async queueRemoveJob(queueType: typeof QUEUE_TYPES[number], jobId: string) { const queue = this.getQueue(queueType); - const job: Bull.Job | null = await queue.getJob(jobId); + const job: Bull.Job | undefined = await queue.getJob(jobId); if (job) { await job.remove(); } @@ -937,7 +943,7 @@ export class QueueService { @bindThis public async queueGetJob(queueType: typeof QUEUE_TYPES[number], jobId: string) { const queue = this.getQueue(queueType); - const job: Bull.Job | null = await queue.getJob(jobId); + const job: Bull.Job | undefined = await queue.getJob(jobId); if (job) { return this.packJobData(job); } else { diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 478438b042..d3ab48e3ff 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -3,11 +3,13 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Inject, Injectable } from '@nestjs/common'; +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { ModuleRef } from '@nestjs/core'; import { DI } from '@/di-symbols.js'; -import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository, NoteThreadMutingsRepository, MiMeta } from '@/models/_.js'; +import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository, MiMeta } from '@/models/_.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { MiRemoteUser, MiUser } from '@/models/User.js'; +import { isLocalUser, isRemoteUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; import { IdService } from '@/core/IdService.js'; import { MiNoteReaction } from '@/models/NoteReaction.js'; @@ -17,13 +19,11 @@ import { NotificationService } from '@/core/NotificationService.js'; import PerUserReactionsChart from '@/core/chart/charts/per-user-reactions.js'; import { emojiRegex } from '@/misc/emoji-regex.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; -import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { bindThis } from '@/decorators.js'; import { UtilityService } from '@/core/UtilityService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; -import { CustomEmojiService } from '@/core/CustomEmojiService.js'; +import { CustomEmojiService, encodeEmojiKey } from '@/core/CustomEmojiService.js'; import { RoleService } from '@/core/RoleService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; import { trackPromise } from '@/misc/promise-tracker.js'; @@ -32,6 +32,7 @@ import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js'; import { CacheService } from '@/core/CacheService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { DataSource } from 'typeorm'; const FALLBACK = '\u2764'; @@ -71,8 +72,12 @@ const isCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@\.)?:$/u; const decodeCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@([\w.-]+))?:$/u; @Injectable() -export class ReactionService { +export class ReactionService implements OnModuleInit { + private roleService: RoleService; + constructor( + private readonly moduleRef: ModuleRef, + @Inject(DI.meta) private meta: MiMeta, @@ -85,9 +90,6 @@ export class ReactionService { @Inject(DI.noteReactionsRepository) private noteReactionsRepository: NoteReactionsRepository, - @Inject(DI.noteThreadMutingsRepository) - private noteThreadMutingsRepository: NoteThreadMutingsRepository, - @Inject(DI.emojisRepository) private emojisRepository: EmojisRepository, @@ -96,9 +98,6 @@ export class ReactionService { private utilityService: UtilityService, private customEmojiService: CustomEmojiService, - private roleService: RoleService, - private userEntityService: UserEntityService, - private noteEntityService: NoteEntityService, private userBlockingService: UserBlockingService, private reactionsBufferingService: ReactionsBufferingService, private idService: IdService, @@ -110,9 +109,15 @@ export class ReactionService { private perUserReactionsChart: PerUserReactionsChart, private readonly cacheService: CacheService, private readonly noteVisibilityService: NoteVisibilityService, + private readonly timeService: TimeService, ) { } + @bindThis + onModuleInit() { + this.roleService = this.moduleRef.get('RoleService'); + } + @bindThis public async create(user: { id: MiUser['id']; host: MiUser['host']; isBot: MiUser['isBot'] }, note: MiNote, _reaction?: string | null) { // Check blocking @@ -144,12 +149,8 @@ export class ReactionService { const reacterHost = this.utilityService.toPunyNullable(user.host); const name = custom[1]; - const emoji = reacterHost == null - ? (await this.customEmojiService.localEmojisCache.fetch()).get(name) - : await this.emojisRepository.findOneBy({ - host: reacterHost, - name, - }); + const emojiKey = encodeEmojiKey({ name, host: reacterHost }); + const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(emojiKey); if (emoji) { if (emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.length === 0 || (await this.roleService.getUserRoles(user.id)).some(r => emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.includes(r.id))) { @@ -223,13 +224,13 @@ export class ReactionService { .execute(); } - this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); + this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); // 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新 if ( Math.random() < 0.3 && note.userId !== user.id && - (Date.now() - this.idService.parse(note.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3 + (this.timeService.now - this.idService.parse(note.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3 ) { const author = await this.cacheService.findUserById(note.userId); if (author.isExplorable) { @@ -256,15 +257,9 @@ export class ReactionService { // カスタム絵文字リアクションだったら絵文字情報も送る const decodedReaction = this.decodeReaction(reaction); - const customEmoji = decodedReaction.name == null ? null : decodedReaction.host == null - ? (await this.customEmojiService.localEmojisCache.fetch()).get(decodedReaction.name) - : await this.emojisRepository.findOne( - { - where: { - name: decodedReaction.name, - host: decodedReaction.host, - }, - }); + const customEmojiKey = decodedReaction.name == null ? null : encodeEmojiKey({ name: decodedReaction.name, host: decodedReaction.host ?? null }); + const customEmoji = customEmojiKey == null ? null : + await this.customEmojiService.emojisByKeyCache.fetchMaybe(customEmojiKey); this.globalEventService.publishNoteStream(note.id, 'reacted', { reaction: decodedReaction.reaction, @@ -290,7 +285,7 @@ export class ReactionService { } //#region 配信 - if (this.userEntityService.isLocalUser(user) && !note.localOnly) { + if (isLocalUser(user) && !note.localOnly) { const content = this.apRendererService.addContext(await this.apRendererService.renderLike(record, note)); const dm = this.apDeliverManagerService.createDeliverManager(user, content); if (note.userHost !== null) { @@ -302,7 +297,7 @@ export class ReactionService { dm.addFollowersRecipe(); } else if (note.visibility === 'specified') { const visibleUsers = await Promise.all(note.visibleUserIds.map(id => this.usersRepository.findOneBy({ id }))); - for (const u of visibleUsers.filter(u => u && this.userEntityService.isRemoteUser(u))) { + for (const u of visibleUsers.filter(u => u && isRemoteUser(u))) { dm.addDirectRecipe(u as MiRemoteUser); } } @@ -345,7 +340,7 @@ export class ReactionService { .execute(); } - this.usersRepository.update({ id: user.id }, { updatedAt: new Date() }); + this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date }); this.globalEventService.publishNoteStream(note.id, 'unreacted', { reaction: this.decodeReaction(exist.reaction).reaction, @@ -353,7 +348,7 @@ export class ReactionService { }); //#region 配信 - if (this.userEntityService.isLocalUser(user) && !note.localOnly) { + if (isLocalUser(user) && !note.localOnly) { const content = this.apRendererService.addContext(this.apRendererService.renderUndo(await this.apRendererService.renderLike(exist, note), user)); const dm = this.apDeliverManagerService.createDeliverManager(user, content); if (note.userHost !== null) { @@ -367,86 +362,102 @@ export class ReactionService { } /** - * - 文字列タイプのレガシーな形式のリアクションを現在の形式に変換する - * - ローカルのリアクションのホストを `@.` にする（`decodeReaction()`の効果） + * @deprecated Use the exported function instead */ - @bindThis - public convertLegacyReaction(reaction: string): string { - reaction = this.decodeReaction(reaction).reaction; - if (Object.keys(legacies).includes(reaction)) return legacies[reaction]; - return reaction; - } + readonly convertLegacyReaction = convertLegacyReaction; - // TODO: 廃止 /** - * - 文字列タイプのレガシーな形式のリアクションを現在の形式に変換する - * - ローカルのリアクションのホストを `@.` にする（`decodeReaction()`の効果） - * - データベース上には存在する「0個のリアクションがついている」という情報を削除する + * @deprecated Use the exported function instead */ - @bindThis - public convertLegacyReactions(reactions: MiNote['reactions']): MiNote['reactions'] { - return Object.entries(reactions) - .filter(([, count]) => { - // `ReactionService.prototype.delete`ではリアクション削除時に、 - // `MiNote['reactions']`のエントリの値をデクリメントしているが、 - // デクリメントしているだけなのでエントリ自体は0を値として持つ形で残り続ける。 - // そのため、この処理がなければ、「0個のリアクションがついている」ということになってしまう。 - return count > 0; - }) - .map(([reaction, count]) => { - const key = this.convertLegacyReaction(reaction); + readonly convertLegacyReactions = convertLegacyReactions; - return [key, count] as const; - }) - .reduce((acc, [key, count]) => { - // unchecked indexed access - const prevCount = acc[key] as number | undefined; + /** + * @deprecated Use the exported function instead + */ + readonly normalize = normalize; - acc[key] = (prevCount ?? 0) + count; + /** + * @deprecated Use the exported function instead + */ + readonly decodeReaction = decodeReaction; +} - return acc; - }, {}); +/** + * - 文字列タイプのレガシーな形式のリアクションを現在の形式に変換する + * - ローカルのリアクションのホストを `@.` にする（`decodeReaction()`の効果） + */ +export function convertLegacyReaction(reaction: string): string { + reaction = decodeReaction(reaction).reaction; + if (Object.keys(legacies).includes(reaction)) return legacies[reaction]; + return reaction; +} + +// TODO: 廃止 +/** + * - 文字列タイプのレガシーな形式のリアクションを現在の形式に変換する + * - ローカルのリアクションのホストを `@.` にする（`decodeReaction()`の効果） + * - データベース上には存在する「0個のリアクションがついている」という情報を削除する + */ +export function convertLegacyReactions(reactions: MiNote['reactions']): MiNote['reactions'] { + return Object.entries(reactions) + .filter(([, count]) => { + // `ReactionService.prototype.delete`ではリアクション削除時に、 + // `MiNote['reactions']`のエントリの値をデクリメントしているが、 + // デクリメントしているだけなのでエントリ自体は0を値として持つ形で残り続ける。 + // そのため、この処理がなければ、「0個のリアクションがついている」ということになってしまう。 + return count > 0; + }) + .map(([reaction, count]) => { + const key = convertLegacyReaction(reaction); + + return [key, count] as const; + }) + .reduce((acc, [key, count]) => { + // unchecked indexed access + const prevCount = acc[key] as number | undefined; + + acc[key] = (prevCount ?? 0) + count; + + return acc; + }, {}); +} + +export function normalize(reaction: string | null): string { + if (reaction == null) return FALLBACK; + + // 文字列タイプのリアクションを絵文字に変換 + if (Object.keys(legacies).includes(reaction)) return legacies[reaction]; + + // Unicode絵文字 + const match = emojiRegex.exec(reaction); + if (match) { + // 合字を含む1つの絵文字 + const unicode = match[0]; + + // 異体字セレクタ除去 + return unicode.match('\u200d') ? unicode : unicode.replace(/\ufe0f/g, ''); } - @bindThis - public normalize(reaction: string | null): string { - if (reaction == null) return FALLBACK; + return FALLBACK; +} - // 文字列タイプのリアクションを絵文字に変換 - if (Object.keys(legacies).includes(reaction)) return legacies[reaction]; +export function decodeReaction(str: string): DecodedReaction { + const custom = str.match(decodeCustomEmojiRegexp); - // Unicode絵文字 - const match = emojiRegex.exec(reaction); - if (match) { - // 合字を含む1つの絵文字 - const unicode = match[0]; - - // 異体字セレクタ除去 - return unicode.match('\u200d') ? unicode : unicode.replace(/\ufe0f/g, ''); - } - - return FALLBACK; - } - - @bindThis - public decodeReaction(str: string): DecodedReaction { - const custom = str.match(decodeCustomEmojiRegexp); - - if (custom) { - const name = custom[1]; - const host = custom[2] ?? null; - - return { - reaction: `:${name}@${host ?? '.'}:`, // ローカル分は@以降を省略するのではなく.にする - name, - host, - }; - } + if (custom) { + const name = custom[1]; + const host = custom[2] ?? null; return { - reaction: str, - name: undefined, - host: undefined, + reaction: `:${name}@${host ?? '.'}:`, // ローカル分は@以降を省略するのではなく.にする + name, + host, }; } + + return { + reaction: str, + name: undefined, + host: undefined, + }; } diff --git a/packages/backend/src/core/ReactionsBufferingService.ts b/packages/backend/src/core/ReactionsBufferingService.ts index b4207c5106..cadaba131d 100644 --- a/packages/backend/src/core/ReactionsBufferingService.ts +++ b/packages/backend/src/core/ReactionsBufferingService.ts @@ -6,6 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; +import { TimeService } from '@/global/TimeService.js'; import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; import type { MiUser, NotesRepository } from '@/models/_.js'; @@ -31,6 +32,8 @@ export class ReactionsBufferingService implements OnApplicationShutdown { @Inject(DI.notesRepository) private notesRepository: NotesRepository, + + private readonly timeService: TimeService, ) { this.redisForSub.on('message', this.onMessage); } @@ -62,7 +65,7 @@ export class ReactionsBufferingService implements OnApplicationShutdown { for (let i = 0; i < currentPairs.length; i++) { pipeline.zadd(`${REDIS_PAIR_PREFIX}:${noteId}`, i, currentPairs[i]); } - pipeline.zadd(`${REDIS_PAIR_PREFIX}:${noteId}`, Date.now(), `${userId}/${reaction}`); + pipeline.zadd(`${REDIS_PAIR_PREFIX}:${noteId}`, this.timeService.now, `${userId}/${reaction}`); pipeline.zremrangebyrank(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -(PER_NOTE_REACTION_USER_PAIR_CACHE_MAX + 1)); await pipeline.exec(); } @@ -144,7 +147,7 @@ export class ReactionsBufferingService implements OnApplicationShutdown { // TODO: scanは重い可能性があるので、別途 bufferedNoteIds を直接Redis上に持っておいてもいいかもしれない @bindThis public async bake(): Promise { - const bufferedNoteIds = []; + const bufferedNoteIds: string[] = []; let cursor = '0'; do { // https://github.com/redis/ioredis#transparent-key-prefixing diff --git a/packages/backend/src/core/RegistryApiService.ts b/packages/backend/src/core/RegistryApiService.ts index 2c7ad4026d..76c7d02a0e 100644 --- a/packages/backend/src/core/RegistryApiService.ts +++ b/packages/backend/src/core/RegistryApiService.ts @@ -11,6 +11,7 @@ import type { MiUser } from '@/models/User.js'; import { IdService } from '@/core/IdService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class RegistryApiService { @@ -20,6 +21,7 @@ export class RegistryApiService { private idService: IdService, private globalEventService: GlobalEventService, + private readonly timeService: TimeService, ) { } @@ -31,7 +33,7 @@ export class RegistryApiService { .insert() .values({ id: this.idService.gen(), - updatedAt: new Date(), + updatedAt: this.timeService.date, userId: userId, domain: domain, scope: scope, diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index 9120de1f9f..afb5b425a1 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -15,10 +15,11 @@ import { DI } from '@/di-symbols.js'; import { deepClone } from '@/misc/clone.js'; import { bindThis } from '@/decorators.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; +import { CacheManagementService, ManagedMemorySingleCache } from '@/global/CacheManagementService.js'; @Injectable() export class RelayService { - private relaysCache: MemorySingleCache; + private readonly relaysCache: ManagedMemorySingleCache; constructor( @Inject(DI.relaysRepository) @@ -28,8 +29,10 @@ export class RelayService { private queueService: QueueService, private systemAccountService: SystemAccountService, private apRendererService: ApRendererService, + + cacheManagementService: CacheManagementService, ) { - this.relaysCache = new MemorySingleCache(1000 * 60 * 10); // 10m + this.relaysCache = cacheManagementService.createMemorySingleCache('relay', 1000 * 60 * 10); // 10m } @bindThis diff --git a/packages/backend/src/core/RemoteUserResolveService.ts b/packages/backend/src/core/RemoteUserResolveService.ts index 4dbc9d6a36..df2c307780 100644 --- a/packages/backend/src/core/RemoteUserResolveService.ts +++ b/packages/backend/src/core/RemoteUserResolveService.ts @@ -16,6 +16,7 @@ import { ILink, WebfingerService } from '@/core/WebfingerService.js'; import { RemoteLoggerService } from '@/core/RemoteLoggerService.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; +import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; @@ -35,6 +36,7 @@ export class RemoteUserResolveService { private remoteLoggerService: RemoteLoggerService, private apDbResolverService: ApDbResolverService, private apPersonService: ApPersonService, + private readonly timeService: TimeService, ) { this.logger = this.remoteLoggerService.logger.createSubLogger('resolve-user'); } @@ -81,10 +83,10 @@ export class RemoteUserResolveService { } // ユーザー情報が古い場合は、WebFingerからやりなおして返す - if (user.lastFetchedAt == null || Date.now() - user.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { + if (user.lastFetchedAt == null || this.timeService.now - user.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { // 繋がらないインスタンスに何回も試行するのを防ぐ, 後続の同様処理の連続試行を防ぐ ため 試行前にも更新する await this.usersRepository.update(user.id, { - lastFetchedAt: new Date(), + lastFetchedAt: this.timeService.date, }); const self = await this.resolveSelf(acctLower); @@ -105,6 +107,7 @@ export class RemoteUserResolveService { }, { uri: self.href, }); + await this.apPersonService.uriPersonCache.delete(user.uri); // Unmap the old URI } this.logger.info(`Corrected URI for ${acctLower} from ${user.uri} to ${self.href}`); diff --git a/packages/backend/src/core/ReversiService.ts b/packages/backend/src/core/ReversiService.ts index b57ab6d9cb..2edba73677 100644 --- a/packages/backend/src/core/ReversiService.ts +++ b/packages/backend/src/core/ReversiService.ts @@ -16,11 +16,11 @@ import type { import type { MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; -import { CacheService } from '@/core/CacheService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { IdService } from '@/core/IdService.js'; -import { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; +import type { NotificationService } from '@/core/NotificationService.js'; import { Serialized } from '@/types.js'; import { ReversiGameEntityService } from './entities/ReversiGameEntityService.js'; import type { OnApplicationShutdown, OnModuleInit } from '@nestjs/common'; @@ -40,16 +40,17 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { @Inject(DI.reversiGamesRepository) private reversiGamesRepository: ReversiGamesRepository, - private cacheService: CacheService, private userEntityService: UserEntityService, private globalEventService: GlobalEventService, private reversiGameEntityService: ReversiGameEntityService, private idService: IdService, + private readonly timeService: TimeService, ) { } + @bindThis async onModuleInit() { - this.notificationService = this.moduleRef.get(NotificationService.name); + this.notificationService = this.moduleRef.get('NotificationService'); } @bindThis @@ -100,8 +101,8 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { // 既にマッチしている対局が無いか探す(3分以内) const games = await this.reversiGamesRepository.find({ where: [ - { id: MoreThan(this.idService.gen(Date.now() - 1000 * 60 * 3)), user1Id: me.id, user2Id: targetUser.id, isStarted: false }, - { id: MoreThan(this.idService.gen(Date.now() - 1000 * 60 * 3)), user1Id: targetUser.id, user2Id: me.id, isStarted: false }, + { id: MoreThan(this.idService.gen(this.timeService.now - 1000 * 60 * 3)), user1Id: me.id, user2Id: targetUser.id, isStarted: false }, + { id: MoreThan(this.idService.gen(this.timeService.now - 1000 * 60 * 3)), user1Id: targetUser.id, user2Id: me.id, isStarted: false }, ], relations: ['user1', 'user2'], order: { id: 'DESC' }, @@ -114,7 +115,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { //#region 相手から既に招待されてないか確認 const invitations = await this.redisClient.zrange( `reversi:matchSpecific:${me.id}`, - Date.now() - INVITATION_TIMEOUT_MS, + this.timeService.now - INVITATION_TIMEOUT_MS, '+inf', 'BYSCORE'); @@ -130,7 +131,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { //#endregion const redisPipeline = this.redisClient.pipeline(); - redisPipeline.zadd(`reversi:matchSpecific:${targetUser.id}`, Date.now(), me.id); + redisPipeline.zadd(`reversi:matchSpecific:${targetUser.id}`, this.timeService.now, me.id); redisPipeline.expire(`reversi:matchSpecific:${targetUser.id}`, 120, 'NX'); await redisPipeline.exec(); @@ -147,8 +148,8 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { // 既にマッチしている対局が無いか探す(3分以内) const games = await this.reversiGamesRepository.find({ where: [ - { id: MoreThan(this.idService.gen(Date.now() - 1000 * 60 * 3)), user1Id: me.id, isStarted: false }, - { id: MoreThan(this.idService.gen(Date.now() - 1000 * 60 * 3)), user2Id: me.id, isStarted: false }, + { id: MoreThan(this.idService.gen(this.timeService.now - 1000 * 60 * 3)), user1Id: me.id, isStarted: false }, + { id: MoreThan(this.idService.gen(this.timeService.now - 1000 * 60 * 3)), user2Id: me.id, isStarted: false }, ], relations: ['user1', 'user2'], order: { id: 'DESC' }, @@ -161,7 +162,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { //#region まず自分宛ての招待を探す const invitations = await this.redisClient.zrange( `reversi:matchSpecific:${me.id}`, - Date.now() - INVITATION_TIMEOUT_MS, + this.timeService.now - INVITATION_TIMEOUT_MS, '+inf', 'BYSCORE'); @@ -202,9 +203,9 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { } else { const redisPipeline = this.redisClient.pipeline(); if (options.noIrregularRules) { - redisPipeline.zadd('reversi:matchAny', Date.now(), me.id + ':noIrregularRules'); + redisPipeline.zadd('reversi:matchAny', this.timeService.now, me.id + ':noIrregularRules'); } else { - redisPipeline.zadd('reversi:matchAny', Date.now(), me.id); + redisPipeline.zadd('reversi:matchAny', this.timeService.now, me.id); } redisPipeline.expire('reversi:matchAny', 15, 'NX'); await redisPipeline.exec(); @@ -225,7 +226,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { @bindThis public async cleanOutdatedGames() { await this.reversiGamesRepository.delete({ - id: LessThan(this.idService.gen(Date.now() - 1000 * 60 * 10)), + id: LessThan(this.idService.gen(this.timeService.now - 1000 * 60 * 10)), isStarted: false, }); } @@ -270,7 +271,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { if (isBothReady) { // 3秒後、両者readyならゲーム開始 - setTimeout(async () => { + this.timeService.startTimer(async () => { const freshGame = await this.get(game.id); if (freshGame == null || freshGame.isStarted || freshGame.isEnded) return; if (!freshGame.user1Ready || !freshGame.user2Ready) return; @@ -324,7 +325,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { const updatedGame = await this.reversiGamesRepository.createQueryBuilder().update() .set({ ...this.getBakeProps(game), - startedAt: new Date(), + startedAt: this.timeService.date, isStarted: true, black: bw, map: game.map, @@ -369,7 +370,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { .set({ ...this.getBakeProps(game), isEnded: true, - endedAt: new Date(), + endedAt: this.timeService.date, winnerId: winnerId, surrenderedUserId: reason === 'surrender' ? (winnerId === game.user1Id ? game.user2Id : game.user1Id) : null, timeoutUserId: reason === 'timeout' ? (winnerId === game.user1Id ? game.user2Id : game.user1Id) : null, @@ -393,7 +394,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { public async getInvitations(user: MiUser): Promise { const invitations = await this.redisClient.zrange( `reversi:matchSpecific:${user.id}`, - Date.now() - INVITATION_TIMEOUT_MS, + this.timeService.now - INVITATION_TIMEOUT_MS, '+inf', 'BYSCORE'); return invitations; @@ -476,7 +477,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { const logs = Reversi.Serializer.deserializeLogs(game.logs); const log = { - time: Date.now(), + time: this.timeService.now, player: myColor, operation: 'put', pos, diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 1418999e9a..6dd768c3c6 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -17,19 +17,24 @@ import type { } from '@/models/_.js'; import { MemoryKVCache, MemorySingleCache } from '@/misc/cache.js'; import type { MiUser } from '@/models/User.js'; +import { isLocalUser, isRemoteUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; -import { CacheService } from '@/core/CacheService.js'; -import type { FollowStats } from '@/core/CacheService.js'; +import type { CacheService, FollowStats } from '@/core/CacheService.js'; import type { RoleCondFormulaValue } from '@/models/Role.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { IdService } from '@/core/IdService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import type { Packed } from '@/misc/json-schema.js'; import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; -import { NotificationService } from '@/core/NotificationService.js'; +import type { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; +import { + CacheManagementService, + type ManagedMemorySingleCache, + type ManagedMemoryKVCache, +} from '@/global/CacheManagementService.js'; import type { OnApplicationShutdown, OnModuleInit } from '@nestjs/common'; import { getCallerId } from '@/misc/attach-caller-id.js'; @@ -115,10 +120,14 @@ export const DEFAULT_POLICIES: RolePolicies = { canViewFederation: true, }; +// TODO cache sync fixes (and maybe events too?) + @Injectable() export class RoleService implements OnApplicationShutdown, OnModuleInit { - private rolesCache: MemorySingleCache; - private roleAssignmentByUserIdCache: MemoryKVCache; + private readonly rolesCache: ManagedMemorySingleCache; + private readonly roleAssignmentByUserIdCache: ManagedMemoryKVCache; + + private cacheService: CacheService; private notificationService: NotificationService; public static AlreadyAssignedError = class extends Error {}; @@ -145,22 +154,25 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { @Inject(DI.roleAssignmentsRepository) private roleAssignmentsRepository: RoleAssignmentsRepository, - private cacheService: CacheService, - private userEntityService: UserEntityService, private globalEventService: GlobalEventService, private idService: IdService, private moderationLogService: ModerationLogService, private fanoutTimelineService: FanoutTimelineService, + private readonly timeService: TimeService, + + cacheManagementService: CacheManagementService, ) { - this.rolesCache = new MemorySingleCache(1000 * 60 * 60); // 1h - this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.rolesCache = cacheManagementService.createMemorySingleCache('roles', 1000 * 60 * 60); // 1h + this.roleAssignmentByUserIdCache = cacheManagementService.createMemoryKVCache('roleAssignment', 1000 * 60 * 5); // 5m // TODO additional cache for final calculation? this.redisForSub.on('message', this.onMessage); } + @bindThis async onModuleInit() { - this.notificationService = this.moduleRef.get(NotificationService.name); + this.notificationService = this.moduleRef.get('NotificationService'); + this.cacheService = this.moduleRef.get('CacheService'); } @bindThis @@ -249,11 +261,11 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { } // ローカルユーザのみ case 'isLocal': { - return this.userEntityService.isLocalUser(user); + return isLocalUser(user); } // リモートユーザのみ case 'isRemote': { - return this.userEntityService.isRemoteUser(user); + return isRemoteUser(user); } // User is from a specific instance case 'isFromInstance': { @@ -294,11 +306,11 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { } // ユーザが作成されてから指定期間経過した case 'createdLessThan': { - return this.idService.parse(user.id).date.getTime() > (Date.now() - (value.sec * 1000)); + return this.idService.parse(user.id).date.getTime() > (this.timeService.now - (value.sec * 1000)); } // ユーザが作成されてから指定期間経っていない case 'createdMoreThan': { - return this.idService.parse(user.id).date.getTime() < (Date.now() - (value.sec * 1000)); + return this.idService.parse(user.id).date.getTime() < (this.timeService.now - (value.sec * 1000)); } // フォロワー数が指定値以下 case 'followersLessThanOrEq': { @@ -398,7 +410,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { @bindThis public async getUserAssigns(userOrId: MiUser | MiUser['id']) { - const now = Date.now(); + const now = this.timeService.now; const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId; let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId })); // 期限切れのロールを除外 @@ -437,7 +449,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { */ @bindThis public async getUserBadgeRoles(userOrId: MiUser | MiUser['id']) { - const now = Date.now(); + const now = this.timeService.now; const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId; let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId })); // 期限切れのロールを除外 @@ -583,7 +595,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { : []; // Setを経由して重複を除去（ユーザIDは重複する可能性があるので） - const now = Date.now(); + const now = this.timeService.now; const resultSet = new Set( assigns .filter(it => @@ -637,7 +649,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { @bindThis public async assign(userId: MiUser['id'], roleId: MiRole['id'], expiresAt: Date | null = null, moderator?: MiUser): Promise { - const now = Date.now(); + const now = this.timeService.now; const role = await this.rolesRepository.findOneByOrFail({ id: roleId }); @@ -665,7 +677,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { }); this.rolesRepository.update(roleId, { - lastUsedAt: new Date(), + lastUsedAt: this.timeService.date, }); this.globalEventService.publishInternalEvent('userRoleAssigned', created); @@ -692,7 +704,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { @bindThis public async unassign(userId: MiUser['id'], roleId: MiRole['id'], moderator?: MiUser): Promise { - const now = new Date(); + const now = this.timeService.date; const existing = await this.roleAssignmentsRepository.findOneBy({ roleId, userId }); if (existing == null) { @@ -744,7 +756,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { @bindThis public async create(values: Partial, moderator?: MiUser): Promise { - const date = new Date(); + const date = this.timeService.date; const created = await this.rolesRepository.insertOne({ id: this.idService.gen(date.getTime()), updatedAt: date, @@ -780,7 +792,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { @bindThis public async update(role: MiRole, params: Partial, moderator?: MiUser): Promise { - const date = new Date(); + const date = this.timeService.date; await this.rolesRepository.update(role.id, { updatedAt: date, ...params, @@ -825,7 +837,6 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { @bindThis public dispose(): void { this.redisForSub.off('message', this.onMessage); - this.roleAssignmentByUserIdCache.dispose(); } @bindThis diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index 968a5dcc0b..955d778015 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -6,24 +6,61 @@ import { URL } from 'node:url'; import * as http from 'node:http'; import * as https from 'node:https'; -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { Upload } from '@aws-sdk/lib-storage'; import { NodeHttpHandler, NodeHttpHandlerOptions } from '@smithy/node-http-handler'; import type { MiMeta } from '@/models/Meta.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; +import { DI } from '@/di-symbols.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import type { InternalEventTypes } from '@/core/GlobalEventService.js'; import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/client-s3'; @Injectable() -export class S3Service { +export class S3Service implements OnApplicationShutdown { + private client?: S3Client; + constructor( + @Inject(DI.meta) + private readonly meta: MiMeta, + private httpRequestService: HttpRequestService, + private readonly internalEventService: InternalEventService, ) { + this.internalEventService.on('metaUpdated', this.onMetaUpdated); } @bindThis - public getS3Client(meta: MiMeta): S3Client { + private onMetaUpdated(body: InternalEventTypes['metaUpdated']): void { + if (this.needsChange(body.before, body.after)) { + this.disposeClient(); + this.client = this.createS3Client(body.after); + } + } + + private needsChange(before: MiMeta | undefined, after: MiMeta): boolean { + if (before == null) return true; + if (before.objectStorageEndpoint !== after.objectStorageEndpoint) return true; + if (before.objectStorageUseSSL !== after.objectStorageUseSSL) return true; + if (before.objectStorageUseProxy !== after.objectStorageUseProxy) return true; + if (before.objectStorageAccessKey !== after.objectStorageAccessKey) return true; + if (before.objectStorageSecretKey !== after.objectStorageSecretKey) return true; + if (before.objectStorageRegion !== after.objectStorageRegion) return true; + if (before.objectStorageUseSSL !== after.objectStorageUseSSL) return true; + if (before.objectStorageS3ForcePathStyle !== after.objectStorageS3ForcePathStyle) return true; + if (before.objectStorageRegion !== after.objectStorageRegion) return true; + return false; + } + + @bindThis + private getS3Client(): S3Client { + return this.client ??= this.createS3Client(this.meta); + } + + @bindThis + private createS3Client(meta: MiMeta): S3Client { const u = meta.objectStorageEndpoint ? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}` : `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent @@ -52,8 +89,8 @@ export class S3Service { } @bindThis - public async upload(meta: MiMeta, input: PutObjectCommandInput) { - const client = this.getS3Client(meta); + public async upload(input: PutObjectCommandInput) { + const client = this.getS3Client(); return new Upload({ client, params: input, @@ -64,8 +101,27 @@ export class S3Service { } @bindThis - public delete(meta: MiMeta, input: DeleteObjectCommandInput) { - const client = this.getS3Client(meta); + public delete(input: DeleteObjectCommandInput) { + const client = this.getS3Client(); return client.send(new DeleteObjectCommand(input)); } + + @bindThis + private disposeClient(): void { + if (this.client) { + this.client.destroy(); + this.client = undefined; + } + } + + @bindThis + private dispose(): void { + this.disposeClient(); + this.internalEventService.off('metaUpdated', this.onMetaUpdated); + } + + @bindThis + onApplicationShutdown() { + this.dispose(); + } } diff --git a/packages/backend/src/core/SignupService.ts b/packages/backend/src/core/SignupService.ts index f1dd0f0503..4291b32f55 100644 --- a/packages/backend/src/core/SignupService.ts +++ b/packages/backend/src/core/SignupService.ts @@ -23,6 +23,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { UserService } from '@/core/UserService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { MetaService } from '@/core/MetaService.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class SignupService { @@ -46,6 +47,7 @@ export class SignupService { private systemAccountService: SystemAccountService, private metaService: MetaService, private usersChart: UsersChart, + private readonly timeService: TimeService, ) { } @@ -150,7 +152,7 @@ export class SignupService { })); await transactionalEntityManager.save(new MiUsedUsername({ - createdAt: new Date(), + createdAt: this.timeService.date, username: username.toLowerCase(), })); }); diff --git a/packages/backend/src/core/SponsorsService.ts b/packages/backend/src/core/SponsorsService.ts index 3769378f5a..551768bdc9 100644 --- a/packages/backend/src/core/SponsorsService.ts +++ b/packages/backend/src/core/SponsorsService.ts @@ -3,12 +3,11 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; -import * as Redis from 'ioredis'; +import { Inject, Injectable } from '@nestjs/common'; import type { MiMeta } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; -import { RedisKVCache } from '@/misc/cache.js'; import { bindThis } from '@/decorators.js'; +import { CacheManagementService, type ManagedRedisKVCache } from '@/global/CacheManagementService.js'; export interface Sponsor { MemberId: number; @@ -34,17 +33,16 @@ export interface Sponsor { } @Injectable() -export class SponsorsService implements OnApplicationShutdown { - private readonly cache: RedisKVCache; +export class SponsorsService { + private readonly cache: ManagedRedisKVCache; constructor( @Inject(DI.meta) private readonly meta: MiMeta, - @Inject(DI.redis) - redisClient: Redis.Redis, + cacheManagementService: CacheManagementService, ) { - this.cache = new RedisKVCache(redisClient, 'sponsors', { + this.cache = cacheManagementService.createRedisKVCache('sponsors', { lifetime: 1000 * 60 * 60, memoryCacheLifetime: 1000 * 60, fetcher: (key) => { @@ -102,9 +100,4 @@ export class SponsorsService implements OnApplicationShutdown { if (forceUpdate) await this.cache.refresh('sharkey'); return this.cache.fetch('sharkey'); } - - @bindThis - public onApplicationShutdown(): void { - this.cache.dispose(); - } } diff --git a/packages/backend/src/core/SystemAccountService.ts b/packages/backend/src/core/SystemAccountService.ts index 1288dc6ffa..1d766f16ae 100644 --- a/packages/backend/src/core/SystemAccountService.ts +++ b/packages/backend/src/core/SystemAccountService.ts @@ -19,12 +19,16 @@ import { bindThis } from '@/decorators.js'; import { generateNativeUserToken } from '@/misc/token.js'; import { IdService } from '@/core/IdService.js'; import { genRsaKeyPair } from '@/misc/gen-key-pair.js'; +import { CacheManagementService, type ManagedMemoryKVCache } from '@/global/CacheManagementService.js'; +import { CacheService } from '@/core/CacheService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { TimeService } from '@/global/TimeService.js'; export const SYSTEM_ACCOUNT_TYPES = ['actor', 'relay', 'proxy'] as const; @Injectable() export class SystemAccountService implements OnApplicationShutdown { - private cache: MemoryKVCache; + private readonly cache: ManagedMemoryKVCache; constructor( @Inject(DI.redisForSub) @@ -46,8 +50,13 @@ export class SystemAccountService implements OnApplicationShutdown { private userProfilesRepository: UserProfilesRepository, private idService: IdService, + private readonly cacheService: CacheService, + private readonly internalEventService: InternalEventService, + private readonly timeService: TimeService, + + cacheManagementService: CacheManagementService, ) { - this.cache = new MemoryKVCache(1000 * 60 * 10); // 10m + this.cache = cacheManagementService.createMemoryKVCache('systemAccount', 1000 * 60 * 10); // 10m this.redisForSub.on('message', this.onMessage); } @@ -84,25 +93,26 @@ export class SystemAccountService implements OnApplicationShutdown { @bindThis public async fetch(type: typeof SYSTEM_ACCOUNT_TYPES[number]): Promise { - const cached = this.cache.get(type); - if (cached) return cached; + // Use local cache to find userId for type + const userId = await this.cache.fetch(type, async () => { + const systemAccount = await this.systemAccountsRepository.findOne({ + where: { type: type }, + select: { userId: true }, + }) as { userId: string } | null; - const systemAccount = await this.systemAccountsRepository.findOne({ - where: { type: type }, - relations: ['user'], + if (systemAccount) { + return systemAccount.userId; + } else { + const created = await this.createCorrespondingUser(type, { + username: `system.${type}`, // NOTE: (できれば避けたいが) . が含まれるかどうかでシステムアカウントかどうかを判定している処理もあるので変えないように + name: this.meta.name, + }); + return created.id; + } }); - if (systemAccount) { - this.cache.set(type, systemAccount.user as MiLocalUser); - return systemAccount.user as MiLocalUser; - } else { - const created = await this.createCorrespondingUser(type, { - username: `system.${type}`, // NOTE: (できれば避けたいが) . が含まれるかどうかでシステムアカウントかどうかを判定している処理もあるので変えないように - name: this.meta.name, - }); - this.cache.set(type, created); - return created; - } + // Get the actual user entity from shared caches. + return await this.cacheService.findLocalUserById(userId); } @bindThis @@ -166,7 +176,7 @@ export class SystemAccountService implements OnApplicationShutdown { }); await transactionalEntityManager.insert(MiUsedUsername, { - createdAt: new Date(), + createdAt: this.timeService.date, username: extra.username.toLowerCase(), }); @@ -192,6 +202,7 @@ export class SystemAccountService implements OnApplicationShutdown { if (Object.keys(updates).length > 0) { await this.usersRepository.update(user.id, updates); + await this.internalEventService.emit('localUserUpdated', { id: user.id }); } const profileUpdates = {} as Partial; @@ -199,12 +210,10 @@ export class SystemAccountService implements OnApplicationShutdown { if (Object.keys(profileUpdates).length > 0) { await this.userProfilesRepository.update(user.id, profileUpdates); + await this.internalEventService.emit('updateUserProfile', { userId: user.id }); } - const updated = await this.usersRepository.findOneByOrFail({ id: user.id }) as MiLocalUser; - this.cache.set(type, updated); - - return updated; + return await this.cacheService.findLocalUserById(user.id); } public async getInstanceActor() { @@ -222,7 +231,6 @@ export class SystemAccountService implements OnApplicationShutdown { @bindThis public dispose(): void { this.redisForSub.off('message', this.onMessage); - this.cache.dispose(); } @bindThis diff --git a/packages/backend/src/core/SystemWebhookService.ts b/packages/backend/src/core/SystemWebhookService.ts index 8239490adc..a5fd89f57b 100644 --- a/packages/backend/src/core/SystemWebhookService.ts +++ b/packages/backend/src/core/SystemWebhookService.ts @@ -18,6 +18,9 @@ import Logger from '@/logger.js'; import { Packed } from '@/misc/json-schema.js'; import { AbuseReportResolveType } from '@/models/AbuseUserReport.js'; import { ModeratorInactivityRemainingTime } from '@/queue/processors/CheckModeratorsActivityProcessorService.js'; +import { CacheManagementService, type ManagedMemorySingleCache } from '@/global/CacheManagementService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; export type AbuseReportPayload = { @@ -50,8 +53,7 @@ export type SystemWebhookPayload = @Injectable() export class SystemWebhookService implements OnApplicationShutdown { - private activeSystemWebhooksFetched = false; - private activeSystemWebhooks: MiSystemWebhook[] = []; + private readonly activeSystemWebhooks: ManagedMemorySingleCache; constructor( @Inject(DI.redisForSub) @@ -62,20 +64,25 @@ export class SystemWebhookService implements OnApplicationShutdown { private queueService: QueueService, private moderationLogService: ModerationLogService, private globalEventService: GlobalEventService, + private readonly internalEventService: InternalEventService, + private readonly timeService: TimeService, + + cacheManagementService: CacheManagementService, ) { - this.redisForSub.on('message', this.onMessage); + this.activeSystemWebhooks = cacheManagementService.createMemorySingleCache('systemWebhooks', 1000 * 60 * 60 * 12); // 12h + + this.internalEventService.on('systemWebhookCreated', this.onWebhookEvent); + this.internalEventService.on('systemWebhookUpdated', this.onWebhookEvent); + this.internalEventService.on('systemWebhookDeleted', this.onWebhookEvent); } @bindThis public async fetchActiveSystemWebhooks() { - if (!this.activeSystemWebhooksFetched) { - this.activeSystemWebhooks = await this.systemWebhooksRepository.findBy({ + return await this.activeSystemWebhooks.fetch(async () => { + return await this.systemWebhooksRepository.findBy({ isActive: true, }); - this.activeSystemWebhooksFetched = true; - } - - return this.activeSystemWebhooks; + }); } /** @@ -124,7 +131,7 @@ export class SystemWebhookService implements OnApplicationShutdown { }); const webhook = await this.systemWebhooksRepository.findOneByOrFail({ id }); - this.globalEventService.publishInternalEvent('systemWebhookCreated', webhook); + await this.internalEventService.emit('systemWebhookCreated', { id: webhook.id }); this.moderationLogService .log(updater, 'createSystemWebhook', { systemWebhookId: webhook.id, @@ -151,7 +158,7 @@ export class SystemWebhookService implements OnApplicationShutdown { ): Promise { const beforeEntity = await this.systemWebhooksRepository.findOneByOrFail({ id: params.id }); await this.systemWebhooksRepository.update(beforeEntity.id, { - updatedAt: new Date(), + updatedAt: this.timeService.date, isActive: params.isActive, name: params.name, on: params.on, @@ -160,7 +167,7 @@ export class SystemWebhookService implements OnApplicationShutdown { }); const afterEntity = await this.systemWebhooksRepository.findOneByOrFail({ id: beforeEntity.id }); - this.globalEventService.publishInternalEvent('systemWebhookUpdated', afterEntity); + await this.internalEventService.emit('systemWebhookUpdated', { id: afterEntity.id }); this.moderationLogService .log(updater, 'updateSystemWebhook', { systemWebhookId: beforeEntity.id, @@ -179,7 +186,7 @@ export class SystemWebhookService implements OnApplicationShutdown { const webhook = await this.systemWebhooksRepository.findOneByOrFail({ id }); await this.systemWebhooksRepository.delete(id); - this.globalEventService.publishInternalEvent('systemWebhookDeleted', webhook); + await this.internalEventService.emit('systemWebhookDeleted', { id: webhook.id }); this.moderationLogService .log(updater, 'deleteSystemWebhook', { systemWebhookId: webhook.id, @@ -211,45 +218,15 @@ export class SystemWebhookService implements OnApplicationShutdown { } @bindThis - private async onMessage(_: string, data: string): Promise { - const obj = JSON.parse(data); - if (obj.channel !== 'internal') { - return; - } - - const { type, body } = obj.message as GlobalEvents['internal']['payload']; - switch (type) { - case 'systemWebhookCreated': { - if (body.isActive) { - this.activeSystemWebhooks.push(MiSystemWebhook.deserialize(body)); - } - break; - } - case 'systemWebhookUpdated': { - if (body.isActive) { - const i = this.activeSystemWebhooks.findIndex(a => a.id === body.id); - if (i > -1) { - this.activeSystemWebhooks[i] = MiSystemWebhook.deserialize(body); - } else { - this.activeSystemWebhooks.push(MiSystemWebhook.deserialize(body)); - } - } else { - this.activeSystemWebhooks = this.activeSystemWebhooks.filter(a => a.id !== body.id); - } - break; - } - case 'systemWebhookDeleted': { - this.activeSystemWebhooks = this.activeSystemWebhooks.filter(a => a.id !== body.id); - break; - } - default: - break; - } + private onWebhookEvent(): void { + this.activeSystemWebhooks.delete(); } @bindThis public dispose(): void { - this.redisForSub.off('message', this.onMessage); + this.internalEventService.off('systemWebhookCreated', this.onWebhookEvent); + this.internalEventService.off('systemWebhookUpdated', this.onWebhookEvent); + this.internalEventService.off('systemWebhookDeleted', this.onWebhookEvent); } @bindThis diff --git a/packages/backend/src/core/TimeService.ts b/packages/backend/src/core/TimeService.ts deleted file mode 100644 index 59c3d4c12b..0000000000 --- a/packages/backend/src/core/TimeService.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { Injectable } from '@nestjs/common'; - -/** - * Provides abstractions to access the current time. - * Exists for unit testing purposes, so that tests can "simulate" any given time for consistency. - */ -@Injectable() -export class TimeService { - /** - * Returns Date.now() - */ - public get now() { - return Date.now(); - } - - /** - * Returns a new Date instance. - */ - public get date() { - return new Date(); - } -} diff --git a/packages/backend/src/core/UpdateInstanceQueue.ts b/packages/backend/src/core/UpdateInstanceQueue.ts index 3fcd215ffa..c136241344 100644 --- a/packages/backend/src/core/UpdateInstanceQueue.ts +++ b/packages/backend/src/core/UpdateInstanceQueue.ts @@ -8,6 +8,7 @@ import { CollapsedQueue } from '@/misc/collapsed-queue.js'; import { bindThis } from '@/decorators.js'; import { MiNote } from '@/models/Note.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import { TimeService } from '@/global/TimeService.js'; type UpdateInstanceJob = { latestRequestReceivedAt: Date, @@ -19,8 +20,9 @@ type UpdateInstanceJob = { export class UpdateInstanceQueue extends CollapsedQueue implements OnApplicationShutdown { constructor( private readonly federatedInstanceService: FederatedInstanceService, + timeService: TimeService, ) { - super(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, (id, job) => this.collapseUpdateInstanceJobs(id, job), (id, job) => this.performUpdateInstance(id, job)); + super(timeService, process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, (id, job) => this.collapseUpdateInstanceJobs(id, job), (id, job) => this.performUpdateInstance(id, job)); } @bindThis @@ -38,7 +40,7 @@ export class UpdateInstanceQueue extends CollapsedQueue l.id)); } @bindThis diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 8470872eac..833ea97193 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -22,14 +22,14 @@ import type { FollowingsRepository, FollowRequestsRepository, InstancesRepositor import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { bindThis } from '@/decorators.js'; -import { UserBlockingService } from '@/core/UserBlockingService.js'; +import type { UserBlockingService } from '@/core/UserBlockingService.js'; import { CacheService } from '@/core/CacheService.js'; import type { Config } from '@/config.js'; import { AccountMoveService } from '@/core/AccountMoveService.js'; import { UtilityService } from '@/core/UtilityService.js'; import type { ThinUser } from '@/queue/types.js'; import { LoggerService } from '@/core/LoggerService.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; import type Logger from '../logger.js'; type Local = MiLocalUser | { @@ -94,6 +94,7 @@ export class UserFollowingService implements OnModuleInit { this.logger = loggerService.getLogger('following/create'); } + @bindThis onModuleInit() { this.userBlockingService = this.moduleRef.get('UserBlockingService'); } diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index d8a67d273b..2a2f1aeab7 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -5,16 +5,19 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; +import { In } from 'typeorm'; import type { MiUser } from '@/models/User.js'; import type { UserKeypairsRepository } from '@/models/_.js'; -import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; +import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import type { InternalEventTypes } from '@/core/GlobalEventService.js'; @Injectable() export class UserKeypairService implements OnApplicationShutdown { - private cache: MemoryKVCache; + public readonly userKeypairCache: ManagedQuantumKVCache; constructor( @Inject(DI.redis) @@ -22,18 +25,37 @@ export class UserKeypairService implements OnApplicationShutdown { @Inject(DI.userKeypairsRepository) private userKeypairsRepository: UserKeypairsRepository, + + private readonly internalEventService: InternalEventService, + + cacheManagementService: CacheManagementService, ) { - this.cache = new MemoryKVCache(1000 * 60 * 60 * 24); // 24h + this.userKeypairCache = cacheManagementService.createQuantumKVCache('userKeypair', { + lifetime: 1000 * 60 * 60, // 1h + fetcher: async userId => await this.userKeypairsRepository.findOneByOrFail({ userId }), + optionalFetcher: async userId => await this.userKeypairsRepository.findOneBy({ userId }), + bulkFetcher: async userIds => { + const keypairs = await this.userKeypairsRepository.findBy({ userId: In(userIds) }); + return keypairs.map(keypair => [keypair.userId, keypair]); + }, + }); + + this.internalEventService.on('userChangeDeletedState', this.onUserDeleted); } @bindThis public async getUserKeypair(userId: MiUser['id']): Promise { - return await this.cache.fetch(userId, () => this.userKeypairsRepository.findOneByOrFail({ userId })); + return await this.userKeypairCache.fetch(userId); + } + + @bindThis + private async onUserDeleted(body: InternalEventTypes['userChangeDeletedState']): Promise { + await this.userKeypairCache.delete(body.id); } @bindThis public dispose(): void { - this.cache.dispose(); + this.internalEventService.off('userChangeDeletedState', this.onUserDeleted); } @bindThis diff --git a/packages/backend/src/core/UserListService.ts b/packages/backend/src/core/UserListService.ts index b4486b9808..01b8173c59 100644 --- a/packages/backend/src/core/UserListService.ts +++ b/packages/backend/src/core/UserListService.ts @@ -6,27 +6,29 @@ import { Inject, Injectable, OnApplicationShutdown, OnModuleInit } from '@nestjs/common'; import * as Redis from 'ioredis'; import { ModuleRef } from '@nestjs/core'; -import type { MiMeta, UserListMembershipsRepository } from '@/models/_.js'; +import { In } from 'typeorm'; +import type { MiMeta, UserListMembershipsRepository, UserListsRepository } from '@/models/_.js'; import type { MiUser } from '@/models/User.js'; import type { MiUserList } from '@/models/UserList.js'; import type { MiUserListMembership } from '@/models/UserListMembership.js'; import { IdService } from '@/core/IdService.js'; -import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; -import { QuantumKVCache } from '@/misc/QuantumKVCache.js'; -import { RoleService } from '@/core/RoleService.js'; +import type { RoleService } from '@/core/RoleService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { CacheService } from '@/core/CacheService.js'; +import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js'; @Injectable() -export class UserListService implements OnApplicationShutdown, OnModuleInit { +export class UserListService implements OnModuleInit { public static TooManyUsersError = class extends Error {}; - public membersCache: QuantumKVCache>; + public readonly userListsCache: ManagedQuantumKVCache; + private roleService: RoleService; constructor( @@ -41,6 +43,9 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { @Inject(DI.userListMembershipsRepository) private userListMembershipsRepository: UserListMembershipsRepository, + @Inject(DI.userListsRepository) + private readonly userListsRepository: UserListsRepository, + @Inject(DI.meta) private readonly meta: MiMeta, @@ -50,52 +55,33 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { private queueService: QueueService, private systemAccountService: SystemAccountService, private readonly internalEventService: InternalEventService, - ) { - this.membersCache = new QuantumKVCache>(this.internalEventService, 'userListMembers', { - lifetime: 1000 * 60 * 30, // 30m - fetcher: (key) => this.userListMembershipsRepository.find({ where: { userListId: key }, select: ['userId'] }).then(xs => new Set(xs.map(x => x.userId))), - }); + private readonly cacheService: CacheService, - this.internalEventService.on('userListMemberAdded', this.onMessage); - this.internalEventService.on('userListMemberRemoved', this.onMessage); + cacheManagementService: CacheManagementService, + ) { + this.userListsCache = cacheManagementService.createQuantumKVCache('userLists', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async id => await this.userListsRepository.findOneByOrFail({ id }), + optionalFetcher: async id => await this.userListsRepository.findOneBy({ id }), + bulkFetcher: async ids => { + const lists = await this.userListsRepository.findBy({ id: In(ids) }); + return lists.map(list => [list.id, list]); + }, + }); } + @bindThis async onModuleInit() { this.roleService = this.moduleRef.get('RoleService'); } - @bindThis - private async onMessage(body: InternalEventTypes[E], type: E): Promise { - { - switch (type) { - case 'userListMemberAdded': { - const { userListId, memberId } = body; - const members = this.membersCache.get(userListId); - if (members) { - members.add(memberId); - } - break; - } - case 'userListMemberRemoved': { - const { userListId, memberId } = body; - const members = this.membersCache.get(userListId); - if (members) { - members.delete(memberId); - } - break; - } - default: - break; - } - } - } - @bindThis public async addMember(target: MiUser, list: MiUserList, me: MiUser) { - const currentCount = await this.userListMembershipsRepository.countBy({ - userListId: list.id, - }); - if (currentCount >= (await this.roleService.getUserPolicies(me.id)).userEachUserListsLimit) { + const [current, policies] = await Promise.all([ + this.cacheService.listUserMembershipsCache.fetch(list.id), + this.roleService.getUserPolicies(me), + ]); + if (current.size >= policies.userEachUserListsLimit) { throw new UserListService.TooManyUsersError(); } @@ -106,13 +92,13 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { userListUserId: list.userId, } as MiUserListMembership); - this.globalEventService.publishInternalEvent('userListMemberAdded', { userListId: list.id, memberId: target.id }); + await this.internalEventService.emit('userListMemberAdded', { userListId: list.id, memberId: target.id }); this.globalEventService.publishUserListStream(list.id, 'userAdded', await this.userEntityService.pack(target)); // このインスタンス内にこのリモートユーザーをフォローしているユーザーがいなくても投稿を受け取るためにダミーのユーザーがフォローしたということにする if (this.userEntityService.isRemoteUser(target) && this.meta.enableProxyAccount) { const proxy = await this.systemAccountService.fetch('proxy'); - this.queueService.createFollowJob([{ from: { id: proxy.id }, to: { id: target.id } }]); + await this.queueService.createFollowJob([{ from: { id: proxy.id }, to: { id: target.id } }]); } } @@ -123,16 +109,15 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { userListId: list.id, }); - this.globalEventService.publishInternalEvent('userListMemberRemoved', { userListId: list.id, memberId: target.id }); + await this.internalEventService.emit('userListMemberRemoved', { userListId: list.id, memberId: target.id }); this.globalEventService.publishUserListStream(list.id, 'userRemoved', await this.userEntityService.pack(target)); } @bindThis public async updateMembership(target: MiUser, list: MiUserList, options: { withReplies?: boolean }) { - const membership = await this.userListMembershipsRepository.findOneBy({ - userId: target.id, - userListId: list.id, - }); + const membership = await this.cacheService.userListMembershipsCache + .fetchMaybe(target.id) + .then(ms => ms?.get(list.id)); if (membership == null) { throw new Error('User is not a member of the list'); @@ -143,17 +128,96 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { }, { withReplies: options.withReplies, }); + + await this.internalEventService.emit('userListMemberUpdated', { userListId: list.id, memberId: target.id }); } @bindThis - public dispose(): void { - this.internalEventService.off('userListMemberAdded', this.onMessage); - this.internalEventService.off('userListMemberRemoved', this.onMessage); - this.membersCache.dispose(); + public async bulkAddMember(target: { id: MiUser['id'] }, memberships: { userListId: MiUserList['id'], withReplies?: boolean }[]): Promise { + const userListIds = memberships.map(m => m.userListId); + const userLists = await this.userListsCache.fetchMany(userListIds); + + // Map userListId => userListUserId + const listUserIds = new Map(userLists.values.map(l => [l.id, l.userId])); + + const toInsert = memberships.map(membership => ({ + id: this.idService.gen(), + userId: target.id, + userListId: membership.userListId, + userListUserId: listUserIds.get(membership.userListId), + withReplies: membership.withReplies, + })); + + await this.userListMembershipsRepository.insert(toInsert); + await this.internalEventService.emit('userListMemberBulkAdded', { + memberId: target.id, + userListIds, + }); + + const targetUser = await this.cacheService.findUserById(target.id); + const packedUser = await this.userEntityService.pack(targetUser); + await Promise.all(memberships.map(async membership => { + await this.globalEventService.publishUserListStream(membership.userListId, 'userAdded', packedUser); + })); + + // このインスタンス内にこのリモートユーザーをフォローしているユーザーがいなくても投稿を受け取るためにダミーのユーザーがフォローしたということにする + if (this.userEntityService.isRemoteUser(targetUser) && this.meta.enableProxyAccount) { + const proxy = await this.systemAccountService.fetch('proxy'); + await this.queueService.createFollowJob([{ from: { id: proxy.id }, to: { id: target.id } }]); + } } @bindThis - public onApplicationShutdown(signal?: string | undefined): void { - this.dispose(); + public async bulkRemoveMember(target: { id: MiUser['id'] }, memberships: { userListId: MiUserList['id'] }[] | MiUserList['id'][]): Promise { + const userListIds = memberships.map(mem => typeof(mem) === 'object' ? mem.userListId : mem); + + await this.userListMembershipsRepository.delete({ + userId: target.id, + userListId: In(userListIds), + }); + + await this.internalEventService.emit('userListMemberBulkRemoved', { + userListIds, + memberId: target.id, + }); + + const targetUser = await this.cacheService.findUserById(target.id); + const packedUser = await this.userEntityService.pack(targetUser); + await Promise.all(userListIds.map(async userListId => { + await this.globalEventService.publishUserListStream(userListId, 'userRemoved', packedUser); + })); + } + + @bindThis + public async bulkUpdateMembership(target: { id: MiUser['id'] }, memberships: { userListId: MiUserList['id'], withReplies: boolean }[]): Promise { + const userListMemberships = await this.cacheService.userListMembershipsCache.fetch(target.id); + const membershipChanges = memberships + .map(mem => { + const old = userListMemberships.get(mem.userListId); + return { + new: mem, + id: old?.id ?? '', + old, + }; + }); + + const toAddReplies = membershipChanges + .filter(mem => mem.old != null && mem.new.withReplies && !mem.old.withReplies) + .map(mem => mem.id); + if (toAddReplies.length > 0) { + await this.userListMembershipsRepository.update({ id: In(toAddReplies) }, { withReplies: true }); + } + + const toRemoveReplies = membershipChanges + .filter(mem => mem.old != null && !mem.new.withReplies && mem.old.withReplies) + .map(mem => mem.id); + if (toRemoveReplies.length > 0) { + await this.userListMembershipsRepository.update({ id: In(toRemoveReplies) }, { withReplies: false }); + } + + await this.internalEventService.emit('userListMemberBulkUpdated', { + memberId: target.id, + userListIds: memberships.map(m => m.userListId), + }); } } diff --git a/packages/backend/src/core/UserSearchService.ts b/packages/backend/src/core/UserSearchService.ts index 4be7bd9bdb..6c6d3a5280 100644 --- a/packages/backend/src/core/UserSearchService.ts +++ b/packages/backend/src/core/UserSearchService.ts @@ -11,12 +11,9 @@ import { bindThis } from '@/decorators.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; import type { Config } from '@/config.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { TimeService } from '@/global/TimeService.js'; import { Packed } from '@/misc/json-schema.js'; -function defaultActiveThreshold() { - return new Date(Date.now() - 1000 * 60 * 60 * 24 * 30); -} - @Injectable() export class UserSearchService { constructor( @@ -36,9 +33,14 @@ export class UserSearchService { private mutingsRepository: MutingsRepository, private userEntityService: UserEntityService, + private readonly timeService: TimeService, ) { } + private defaultActiveThreshold() { + return new Date(this.timeService.now - 1000 * 60 * 60 * 24 * 30); + } + /** * ユーザ名とホスト名によるユーザ検索を行う. * @@ -120,7 +122,7 @@ export class UserSearchService { }, ) { // デフォルト30日以内に更新されたユーザーをアクティブユーザーとする - const activeThreshold = params.activeThreshold ?? defaultActiveThreshold(); + const activeThreshold = params.activeThreshold ?? this.defaultActiveThreshold(); const followingUserQuery = this.followingsRepository.createQueryBuilder('following') .select('following.followeeId') @@ -166,7 +168,7 @@ export class UserSearchService { activeThreshold?: Date, }) { // デフォルト30日以内に更新されたユーザーをアクティブユーザーとする - const activeThreshold = params.activeThreshold ?? defaultActiveThreshold(); + const activeThreshold = params.activeThreshold ?? this.defaultActiveThreshold(); const activeUserQuery = this.generateUserQueryBuilder(params) .andWhere(new Brackets(qb => { @@ -218,7 +220,7 @@ export class UserSearchService { offset: number; origin: 'local' | 'remote' | 'combined'; }> = {}) { - const activeThreshold = new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)); // 30日 + const activeThreshold = new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 30)); // 30日 const isUsername = query.startsWith('@') && !query.includes(' ') && query.indexOf('@', 1) === -1; diff --git a/packages/backend/src/core/UserService.ts b/packages/backend/src/core/UserService.ts index 4a04910105..34c343712c 100644 --- a/packages/backend/src/core/UserService.ts +++ b/packages/backend/src/core/UserService.ts @@ -11,6 +11,7 @@ import { bindThis } from '@/decorators.js'; import { SystemWebhookService } from '@/core/SystemWebhookService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { CacheService } from '@/core/CacheService.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class UserService { @@ -22,6 +23,7 @@ export class UserService { private systemWebhookService: SystemWebhookService, private userEntityService: UserEntityService, private readonly cacheService: CacheService, + private readonly timeService: TimeService, ) { } @@ -30,7 +32,7 @@ export class UserService { if (user.isHibernated) { const result = await this.usersRepository.createQueryBuilder().update() .set({ - lastActiveDate: new Date(), + lastActiveDate: this.timeService.date, }) .where('id = :id', { id: user.id }) .returning('*') @@ -54,7 +56,7 @@ export class UserService { } } else { this.usersRepository.update(user.id, { - lastActiveDate: new Date(), + lastActiveDate: this.timeService.date, }); } } diff --git a/packages/backend/src/core/UserSuspendService.ts b/packages/backend/src/core/UserSuspendService.ts index 5868ba6678..4e42c24383 100644 --- a/packages/backend/src/core/UserSuspendService.ts +++ b/packages/backend/src/core/UserSuspendService.ts @@ -21,7 +21,7 @@ import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { trackPromise } from '@/misc/promise-tracker.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; @Injectable() export class UserSuspendService { diff --git a/packages/backend/src/core/UserWebhookService.ts b/packages/backend/src/core/UserWebhookService.ts index 2f79eb429a..24a519bc5c 100644 --- a/packages/backend/src/core/UserWebhookService.ts +++ b/packages/backend/src/core/UserWebhookService.ts @@ -9,9 +9,11 @@ import { MiUser, type WebhooksRepository } from '@/models/_.js'; import { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; -import { GlobalEvents } from '@/core/GlobalEventService.js'; +import type { InternalEventTypes } from '@/core/GlobalEventService.js'; import type { Packed } from '@/misc/json-schema.js'; import { QueueService } from '@/core/QueueService.js'; +import { CacheManagementService, type ManagedMemorySingleCache } from '@/global/CacheManagementService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; export type UserWebhookPayload = @@ -27,8 +29,7 @@ export type UserWebhookPayload = @Injectable() export class UserWebhookService implements OnApplicationShutdown { - private activeWebhooksFetched = false; - private activeWebhooks: MiWebhook[] = []; + private readonly activeWebhooks: ManagedMemorySingleCache; constructor( @Inject(DI.redisForSub) @@ -36,20 +37,24 @@ export class UserWebhookService implements OnApplicationShutdown { @Inject(DI.webhooksRepository) private webhooksRepository: WebhooksRepository, private queueService: QueueService, + private readonly internalEventService: InternalEventService, + + cacheManagementService: CacheManagementService, ) { - this.redisForSub.on('message', this.onMessage); + this.activeWebhooks = cacheManagementService.createMemorySingleCache('userWebhooks', 1000 * 60 * 60 * 12); // 12h + + this.internalEventService.on('webhookCreated', this.onWebhookEvent); + this.internalEventService.on('webhookUpdated', this.onWebhookEvent); + this.internalEventService.on('webhookDeleted', this.onWebhookEvent); } @bindThis public async getActiveWebhooks() { - if (!this.activeWebhooksFetched) { - this.activeWebhooks = await this.webhooksRepository.findBy({ + return await this.activeWebhooks.fetch(async () => { + return await this.webhooksRepository.findBy({ active: true, }); - this.activeWebhooksFetched = true; - } - - return this.activeWebhooks; + }); } /** @@ -97,57 +102,51 @@ export class UserWebhookService implements OnApplicationShutdown { } @bindThis - private async onMessage(_: string, data: string): Promise { - const obj = JSON.parse(data); - if (obj.channel !== 'internal') { + private async onWebhookEvent(body: InternalEventTypes[E], type: E): Promise { + const cache = this.activeWebhooks.get(); + if (!cache) { return; } - const { type, body } = obj.message as GlobalEvents['internal']['payload']; switch (type) { case 'webhookCreated': { - if (body.active) { - this.activeWebhooks.push({ // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - latestSentAt: body.latestSentAt ? new Date(body.latestSentAt) : null, - user: null, // joinなカラムは通常取ってこないので - }); + // Add + const webhook = await this.webhooksRepository.findOneBy({ id: body.id }); + if (webhook) { + cache.push(webhook); } break; } case 'webhookUpdated': { - if (body.active) { - const i = this.activeWebhooks.findIndex(a => a.id === body.id); - if (i > -1) { - this.activeWebhooks[i] = { // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - latestSentAt: body.latestSentAt ? new Date(body.latestSentAt) : null, - user: null, // joinなカラムは通常取ってこないので - }; - } else { - this.activeWebhooks.push({ // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい - ...body, - latestSentAt: body.latestSentAt ? new Date(body.latestSentAt) : null, - user: null, // joinなカラムは通常取ってこないので - }); - } - } else { - this.activeWebhooks = this.activeWebhooks.filter(a => a.id !== body.id); + // Delete + const index = cache.findIndex(webhook => webhook.id === body.id); + if (index > -1) { + cache.splice(index, 1); + } + + // Add + const webhook = await this.webhooksRepository.findOneBy({ id: body.id }); + if (webhook) { + cache.push(webhook); } break; } case 'webhookDeleted': { - this.activeWebhooks = this.activeWebhooks.filter(a => a.id !== body.id); + // Delete + const index = cache.findIndex(webhook => webhook.id === body.id); + if (index > -1) { + cache.splice(index, 1); + } break; } - default: - break; } } @bindThis public dispose(): void { - this.redisForSub.off('message', this.onMessage); + this.internalEventService.off('webhookCreated', this.onWebhookEvent); + this.internalEventService.off('webhookUpdated', this.onWebhookEvent); + this.internalEventService.off('webhookDeleted', this.onWebhookEvent); } @bindThis diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index 11606e3184..8ebf7e6f52 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -14,7 +14,7 @@ import { bindThis } from '@/decorators.js'; import type { MiMeta, SoftwareSuspension } from '@/models/Meta.js'; import type { MiInstance } from '@/models/Instance.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { EnvService } from '@/core/EnvService.js'; +import { EnvService } from '@/global/EnvService.js'; @Injectable() export class UtilityService { diff --git a/packages/backend/src/core/WebAuthnService.ts b/packages/backend/src/core/WebAuthnService.ts index afd1d68ce4..1e31dff257 100644 --- a/packages/backend/src/core/WebAuthnService.ts +++ b/packages/backend/src/core/WebAuthnService.ts @@ -18,6 +18,7 @@ import { bindThis } from '@/decorators.js'; import { MiUser } from '@/models/_.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { TimeService } from '@/global/TimeService.js'; import Logger from '@/logger.js'; import type { AuthenticationResponseJSON, @@ -44,6 +45,9 @@ export class WebAuthnService { @Inject(DI.userSecurityKeysRepository) private userSecurityKeysRepository: UserSecurityKeysRepository, + + private readonly timeService: TimeService, + loggerService: LoggerService, ) { this.logger = loggerService.getLogger('web-authn'); @@ -239,7 +243,7 @@ export class WebAuthnService { await this.userSecurityKeysRepository.update({ id: response.id, }, { - lastUsed: new Date(), + lastUsed: this.timeService.date, counter: authenticationInfo.newCounter, credentialDeviceType: authenticationInfo.credentialDeviceType, credentialBackedUp: authenticationInfo.credentialBackedUp, @@ -321,7 +325,7 @@ export class WebAuthnService { id: response.id, userId: userId, }, { - lastUsed: new Date(), + lastUsed: this.timeService.date, counter: authenticationInfo.newCounter, credentialDeviceType: authenticationInfo.credentialDeviceType, credentialBackedUp: authenticationInfo.credentialBackedUp, diff --git a/packages/backend/src/core/WebhookTestService.ts b/packages/backend/src/core/WebhookTestService.ts index dd90d147c0..3d5b94254b 100644 --- a/packages/backend/src/core/WebhookTestService.ts +++ b/packages/backend/src/core/WebhookTestService.ts @@ -14,16 +14,17 @@ import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { type UserWebhookPayload, UserWebhookService } from '@/core/UserWebhookService.js'; import { QueueService } from '@/core/QueueService.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import { ModeratorInactivityRemainingTime } from '@/queue/processors/CheckModeratorsActivityProcessorService.js'; const oneDayMillis = 24 * 60 * 60 * 1000; -function generateDummyUser(override?: Partial): MiUser { +function generateDummyUser(now: number, override?: Partial): MiUser { return { id: 'dummy-user-1', - updatedAt: new Date(Date.now() - oneDayMillis * 7), - lastFetchedAt: new Date(Date.now() - oneDayMillis * 5), - lastActiveDate: new Date(Date.now() - oneDayMillis * 3), + updatedAt: new Date(now - oneDayMillis * 7), + lastFetchedAt: new Date(now - oneDayMillis * 5), + lastActiveDate: new Date(now - oneDayMillis * 3), hideOnlineStatus: false, username: 'dummy1', usernameLower: 'dummy1', @@ -132,31 +133,34 @@ function generateDummyNote(override?: Partial): MiNote { }; } -const dummyUser1 = generateDummyUser(); -const dummyUser2 = generateDummyUser({ - id: 'dummy-user-2', - updatedAt: new Date(Date.now() - oneDayMillis * 30), - lastFetchedAt: new Date(Date.now() - oneDayMillis), - lastActiveDate: new Date(Date.now() - oneDayMillis), - username: 'dummy2', - usernameLower: 'dummy2', - name: 'DummyUser2', - followersCount: 40, - followingCount: 50, - notesCount: 900, -}); -const dummyUser3 = generateDummyUser({ - id: 'dummy-user-3', - updatedAt: new Date(Date.now() - oneDayMillis * 15), - lastFetchedAt: new Date(Date.now() - oneDayMillis * 2), - lastActiveDate: new Date(Date.now() - oneDayMillis * 2), - username: 'dummy3', - usernameLower: 'dummy3', - name: 'DummyUser3', - followersCount: 60, - followingCount: 70, - notesCount: 15900, -}); +function makeDummyUsers(now: number) { + const dummyUser1 = generateDummyUser(now); + const dummyUser2 = generateDummyUser(now, { + id: 'dummy-user-2', + updatedAt: new Date(now - oneDayMillis * 30), + lastFetchedAt: new Date(now - oneDayMillis), + lastActiveDate: new Date(now - oneDayMillis), + username: 'dummy2', + usernameLower: 'dummy2', + name: 'DummyUser2', + followersCount: 40, + followingCount: 50, + notesCount: 900, + }); + const dummyUser3 = generateDummyUser(now, { + id: 'dummy-user-3', + updatedAt: new Date(now - oneDayMillis * 15), + lastFetchedAt: new Date(now - oneDayMillis * 2), + lastActiveDate: new Date(now - oneDayMillis * 2), + username: 'dummy3', + usernameLower: 'dummy3', + name: 'DummyUser3', + followersCount: 60, + followingCount: 70, + notesCount: 15900, + }); + return { dummyUser1, dummyUser2, dummyUser3 }; +} @Injectable() export class WebhookTestService { @@ -169,6 +173,7 @@ export class WebhookTestService { private systemWebhookService: SystemWebhookService, private queueService: QueueService, private readonly idService: IdService, + private readonly timeService: TimeService, ) { } @@ -207,6 +212,8 @@ export class WebhookTestService { this.queueService.userWebhookDeliver(merged, type, contents, { attempts: 1 }); }; + const { dummyUser1, dummyUser2, dummyUser3 } = makeDummyUsers(this.timeService.now); + const dummyNote1 = generateDummyNote({ userId: dummyUser1.id, user: dummyUser1, @@ -311,6 +318,8 @@ export class WebhookTestService { this.queueService.systemWebhookDeliver(merged, type, contents, { attempts: 1 }); }; + const { dummyUser1, dummyUser2, dummyUser3 } = makeDummyUsers(this.timeService.now); + switch (params.type) { case 'abuseReport': { send('abuseReport', await this.generateAbuseReport({ @@ -396,13 +405,13 @@ export class WebhookTestService { return { id: note.id, threadId: note.threadId ?? note.id, - createdAt: new Date().toISOString(), + createdAt: this.timeService.date.toISOString(), deletedAt: null, text: note.text, cw: note.cw, userId: note.userId, userHost: note.userHost ?? null, - user: await this.toPackedUserLite(note.user ?? generateDummyUser()), + user: await this.toPackedUserLite(note.user ?? generateDummyUser(this.timeService.now)), replyId: note.replyId, renoteId: note.renoteId, isHidden: false, @@ -486,7 +495,7 @@ export class WebhookTestService { uri: null, movedTo: null, alsoKnownAs: [], - createdAt: new Date().toISOString(), + createdAt: this.timeService.date.toISOString(), updatedAt: user.updatedAt?.toISOString() ?? null, lastFetchedAt: user.lastFetchedAt?.toISOString() ?? null, bannerUrl: user.bannerId == null ? null : user.bannerUrl, diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index e9e0dde9cd..a5eb851d6c 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -107,15 +107,25 @@ export class ApDbResolverService implements OnApplicationShutdown { if (parsed.local) { if (parsed.type !== 'users') return null; - return await this.cacheService.userByIdCache.fetchMaybe( - parsed.id, - () => this.usersRepository.findOneBy({ id: parsed.id, isDeleted: false }).then(x => x ?? undefined), - ) as MiLocalUser | undefined ?? null; + const u = await this.cacheService.findOptionalUserById(parsed.id); + + if (u == null || u.isDeleted) { + return null; + } + + return u as MiLocalUser | MiRemoteUser; } else { - return await this.cacheService.uriPersonCache.fetch( - parsed.uri, - () => this.usersRepository.findOneBy({ uri: parsed.uri, isDeleted: false }), - ) as MiRemoteUser | null; + const uid = await this.apPersonService.uriPersonCache.fetchMaybe(parsed.uri); + if (uid == null) { + return null; + } + + const u = await this.cacheService.findOptionalUserById(uid); + if (u == null || u.isDeleted) { + return null; + } + + return u as MiLocalUser | MiRemoteUser; } } diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 8aaa229466..91eba793d6 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -39,6 +39,7 @@ import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataServic import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; import { CacheService } from '@/core/CacheService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; +import { TimeService } from '@/global/TimeService.js'; import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; @@ -102,6 +103,7 @@ export class ApInboxService { private readonly updateInstanceQueue: UpdateInstanceQueue, private readonly cacheService: CacheService, private readonly noteVisibilityService: NoteVisibilityService, + private readonly timeService: TimeService, ) { this.logger = this.apLoggerService.logger; } @@ -150,7 +152,7 @@ export class ApInboxService { // ついでにリモートユーザーの情報が古かったら更新しておく if (actor.uri) { - if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { + if (actor.lastFetchedAt == null || this.timeService.now - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { setImmediate(() => { // 同一ユーザーの情報を再度処理するので、使用済みのresolverを再利用してはいけない this.apPersonService.updatePerson(actor.uri) @@ -397,7 +399,7 @@ export class ApInboxService { uri, }); } finally { - unlock(); + await unlock(); } } @@ -431,7 +433,7 @@ export class ApInboxService { if (i == null) return; this.updateInstanceQueue.enqueue(i.id, { - latestRequestReceivedAt: new Date(), + latestRequestReceivedAt: this.timeService.date, shouldUnsuspend: i.suspensionState === 'autoSuspendedForNotResponding', }); @@ -446,7 +448,7 @@ export class ApInboxService { return await this.performOneActivity(actor, activity, resolver) .finally(() => { // Update user (adapted from performActivity) - if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { + if (actor.lastFetchedAt == null || this.timeService.now - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { setImmediate(() => { // Don't re-use the resolver, or it may throw recursion errors. // Instead, create a new resolver with an appropriately-reduced recursion limit. @@ -547,7 +549,7 @@ export class ApInboxService { await this.apNoteService.createNote(note, actor, resolver, silent); return 'ok'; } finally { - unlock(); + await unlock(); } } @@ -632,7 +634,7 @@ export class ApInboxService { await this.noteDeleteService.delete(actor, note); return 'ok: note deleted'; } finally { - unlock(); + await unlock(); } } diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 25ad0852cb..6fc6786bf3 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -35,6 +35,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { CacheService } from '@/core/CacheService.js'; import { isPureRenote, isQuote, isRenote } from '@/misc/is-renote.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import { TimeService } from '@/global/TimeService.js'; import { JsonLdService } from './JsonLdService.js'; import { ApMfmService } from './ApMfmService.js'; import { CONTEXT } from './misc/contexts.js'; @@ -80,6 +81,7 @@ export class ApRendererService { private readonly queryService: QueryService, private readonly cacheService: CacheService, private readonly federatedInstanceService: FederatedInstanceService, + private readonly timeService: TimeService, ) { } @@ -174,7 +176,7 @@ export class ApRendererService { type: 'Delete', actor: this.userEntityService.genLocalUserUri(user.id), object, - published: new Date().toISOString(), + published: this.timeService.date.toISOString(), }; } @@ -196,7 +198,7 @@ export class ApRendererService { id: `${this.config.url}/emojis/${emoji.name}`, type: 'Emoji', name: `:${emoji.name}:`, - updated: emoji.updatedAt != null ? emoji.updatedAt.toISOString() : new Date().toISOString(), + updated: emoji.updatedAt != null ? emoji.updatedAt.toISOString() : this.timeService.date.toISOString(), icon: { type: 'Image', mediaType: emoji.type ?? 'image/png', @@ -359,7 +361,7 @@ export class ApRendererService { if (reaction.startsWith(':')) { const name = reaction.replaceAll(':', ''); - const emoji = (await this.customEmojiService.localEmojisCache.fetch()).get(name); + const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(name); if (emoji && !emoji.localOnly) object.tag = [this.renderEmoji(emoji)]; } @@ -534,7 +536,7 @@ export class ApRendererService { const asPoll = poll ? { type: 'Question', - [poll.expiresAt && poll.expiresAt < new Date() ? 'closed' : 'endTime']: poll.expiresAt, + [poll.expiresAt && poll.expiresAt < this.timeService.date ? 'closed' : 'endTime']: poll.expiresAt, [poll.multiple ? 'anyOf' : 'oneOf']: poll.choices.map((text, i) => ({ type: 'Note', name: text, @@ -755,21 +757,21 @@ export class ApRendererService { ...(id ? { id } : {}), actor: this.userEntityService.genLocalUserUri(user.id), object, - published: new Date().toISOString(), + published: this.timeService.date.toISOString(), }; } @bindThis public renderUpdate(object: IObject, user: { id: MiUser['id'] }): IUpdate { // Deterministic activity IDs to allow de-duplication by remote instances - const updatedAt = object.updated ? new Date(object.updated).getTime() : Date.now(); + const updatedAt = object.updated ? new Date(object.updated).getTime() : this.timeService.now; return { id: `${this.config.url}/users/${user.id}#updates/${updatedAt}`, actor: this.userEntityService.genLocalUserUri(user.id), type: 'Update', to: ['https://www.w3.org/ns/activitystreams#Public'], object, - published: new Date().toISOString(), + published: this.timeService.date.toISOString(), }; } @@ -780,7 +782,7 @@ export class ApRendererService { actor: this.userEntityService.genLocalUserUri(user.id), type: 'Create', to: [pollOwner.uri], - published: new Date().toISOString(), + published: this.timeService.date.toISOString(), object: { id: `${this.config.url}/users/${user.id}#votes/${vote.id}`, type: 'Note', @@ -948,12 +950,10 @@ export class ApRendererService { } @bindThis - private async getEmojis(names: string[]): Promise { + private async getEmojis(names: string[]): Promise { if (names.length === 0) return []; - const allEmojis = await this.customEmojiService.localEmojisCache.fetch(); - const emojis = names.map(name => allEmojis.get(name)).filter(x => x != null); - - return emojis; + const emojis = await this.customEmojiService.emojisByKeyCache.fetchMany(names); + return emojis.values; } } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 7669ce9669..c52f9475f0 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -14,6 +14,7 @@ import { UserKeypairService } from '@/core/UserKeypairService.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; @@ -40,7 +41,7 @@ type PrivateKey = { }; export class ApRequestCreator { - static createSignedPost(args: { key: PrivateKey, url: string, body: string, digest?: string, additionalHeaders: Record }): Signed { + static createSignedPost(args: { key: PrivateKey, url: string, body: string, digest?: string, additionalHeaders: Record, now: Date | string | number }): Signed { const u = new URL(args.url); const digestHeader = args.digest ?? this.createDigest(args.body); @@ -48,7 +49,7 @@ export class ApRequestCreator { url: u.href, method: 'POST', headers: this.#objectAssignWithLcKey({ - 'Date': new Date().toUTCString(), + 'Date': new Date(args.now).toUTCString(), 'Host': u.host, 'Content-Type': 'application/activity+json', 'Digest': digestHeader, @@ -69,7 +70,7 @@ export class ApRequestCreator { return `SHA-256=${crypto.createHash('sha256').update(body).digest('base64')}`; } - static createSignedGet(args: { key: PrivateKey, url: string, additionalHeaders: Record }): Signed { + static createSignedGet(args: { key: PrivateKey, url: string, additionalHeaders: Record, now: Date | string | number }): Signed { const u = new URL(args.url); const request: Request = { @@ -77,7 +78,7 @@ export class ApRequestCreator { method: 'GET', headers: this.#objectAssignWithLcKey({ 'Accept': 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - 'Date': new Date().toUTCString(), + 'Date': new Date(args.now).toUTCString(), 'Host': new URL(args.url).host, }, args.additionalHeaders), }; @@ -150,6 +151,7 @@ export class ApRequestService { private httpRequestService: HttpRequestService, private loggerService: LoggerService, private readonly apUtilityService: ApUtilityService, + private readonly timeService: TimeService, ) { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる @@ -171,6 +173,7 @@ export class ApRequestService { digest, additionalHeaders: { }, + now: this.timeService.now, }); await this.httpRequestService.send(url, { @@ -200,6 +203,7 @@ export class ApRequestService { url, additionalHeaders: { }, + now: this.timeService.now, }); const res = await this.httpRequestService.send(url, { diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index d53e265d36..f6178ace1a 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -30,7 +30,7 @@ import { ApRequestService } from './ApRequestService.js'; import type { IObject, ApObject, IAnonymousObject } from './type.js'; export class Resolver { - private history: Set; + protected readonly history: Set; private user?: MiLocalUser; private logger: Logger; @@ -52,7 +52,7 @@ export class Resolver { private readonly apLogService: ApLogService, private readonly apUtilityService: ApUtilityService, private readonly cacheService: CacheService, - private recursionLimit = 256, + protected readonly recursionLimit = 256, ) { this.history = new Set(); this.logger = this.loggerService.getLogger('ap-resolve'); @@ -410,36 +410,36 @@ export class Resolver { export class ApResolverService { constructor( @Inject(DI.config) - private config: Config, + protected readonly config: Config, @Inject(DI.meta) - private meta: MiMeta, + protected readonly meta: MiMeta, @Inject(DI.usersRepository) - private usersRepository: UsersRepository, + protected readonly usersRepository: UsersRepository, @Inject(DI.notesRepository) - private notesRepository: NotesRepository, + protected readonly notesRepository: NotesRepository, @Inject(DI.pollsRepository) - private pollsRepository: PollsRepository, + protected readonly pollsRepository: PollsRepository, @Inject(DI.noteReactionsRepository) - private noteReactionsRepository: NoteReactionsRepository, + protected readonly noteReactionsRepository: NoteReactionsRepository, @Inject(DI.followRequestsRepository) - private followRequestsRepository: FollowRequestsRepository, + protected readonly followRequestsRepository: FollowRequestsRepository, - private utilityService: UtilityService, - private systemAccountService: SystemAccountService, - private apRequestService: ApRequestService, - private httpRequestService: HttpRequestService, - private apRendererService: ApRendererService, - private apDbResolverService: ApDbResolverService, - private loggerService: LoggerService, - private readonly apLogService: ApLogService, - private readonly apUtilityService: ApUtilityService, - private readonly cacheService: CacheService, + protected readonly utilityService: UtilityService, + protected readonly systemAccountService: SystemAccountService, + protected readonly apRequestService: ApRequestService, + protected readonly httpRequestService: HttpRequestService, + protected readonly apRendererService: ApRendererService, + protected readonly apDbResolverService: ApDbResolverService, + protected readonly loggerService: LoggerService, + protected readonly apLogService: ApLogService, + protected readonly apUtilityService: ApUtilityService, + protected readonly cacheService: CacheService, ) { } diff --git a/packages/backend/src/core/activitypub/JsonLdService.ts b/packages/backend/src/core/activitypub/JsonLdService.ts index 8f150ab201..8e14e0909f 100644 --- a/packages/backend/src/core/activitypub/JsonLdService.ts +++ b/packages/backend/src/core/activitypub/JsonLdService.ts @@ -11,6 +11,7 @@ import { bindThis } from '@/decorators.js'; import Logger from '@/logger.js'; import { LoggerService } from '@/core/LoggerService.js'; import { StatusError } from '@/misc/status-error.js'; +import { TimeService } from '@/global/TimeService.js'; import { CONTEXT, PRELOADED_CONTEXTS } from './misc/contexts.js'; import { validateContentTypeSetAsJsonLD } from './misc/validator.js'; import type { ContextDefinition, JsonLdDocument } from 'jsonld'; @@ -56,6 +57,8 @@ export class JsonLdService { constructor( private httpRequestService: HttpRequestService, + private readonly timeService: TimeService, + loggerService: LoggerService, ) { this.logger = loggerService.getLogger('json-ld'); @@ -73,7 +76,7 @@ export class JsonLdService { type: 'RsaSignature2017', creator, nonce: crypto.randomBytes(16).toString('hex'), - created: (created ?? new Date()).toISOString(), + created: (created ?? this.timeService.date).toISOString(), }; if (domain) { diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 652efd46b2..2c40a6e408 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -3,10 +3,11 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { forwardRef, Inject, Injectable } from '@nestjs/common'; +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; import { In } from 'typeorm'; import { UnrecoverableError } from 'bullmq'; import promiseLimit from 'promise-limit'; +import { ModuleRef } from '@nestjs/core'; import { DI } from '@/di-symbols.js'; import type { UsersRepository, PollsRepository, EmojisRepository, NotesRepository, MiMeta } from '@/models/_.js'; import type { Config } from '@/config.js'; @@ -31,6 +32,8 @@ import { renderInlineError } from '@/misc/render-inline-error.js'; import { extractMediaFromHtml } from '@/core/activitypub/misc/extract-media-from-html.js'; import { extractMediaFromMfm } from '@/core/activitypub/misc/extract-media-from-mfm.js'; import { getContentByType } from '@/core/activitypub/misc/get-content-by-type.js'; +import { CustomEmojiService, encodeEmojiKey, isValidEmojiName } from '@/core/CustomEmojiService.js'; +import { TimeService } from '@/global/TimeService.js'; import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument, isLink } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { ApMfmService } from '../ApMfmService.js'; @@ -38,19 +41,22 @@ import { ApDbResolverService } from '../ApDbResolverService.js'; import { ApResolverService } from '../ApResolverService.js'; import { ApAudienceService } from '../ApAudienceService.js'; import { ApUtilityService } from '../ApUtilityService.js'; -import { ApPersonService } from './ApPersonService.js'; import { extractApHashtags } from './tag.js'; import { ApMentionService } from './ApMentionService.js'; import { ApQuestionService } from './ApQuestionService.js'; import { ApImageService } from './ApImageService.js'; +import type { ApPersonService } from './ApPersonService.js'; import type { Resolver } from '../ApResolverService.js'; -import type { IObject, IPost } from '../type.js'; +import type { IObject, IPost, IApEmoji } from '../type.js'; @Injectable() -export class ApNoteService { +export class ApNoteService implements OnModuleInit { + private apPersonService: ApPersonService; private logger: Logger; constructor( + private readonly moduleRef: ModuleRef, + @Inject(DI.config) private config: Config, @@ -73,10 +79,6 @@ export class ApNoteService { private apMfmService: ApMfmService, private apResolverService: ApResolverService, - // 循環参照のため / for circular dependency - @Inject(forwardRef(() => ApPersonService)) - private apPersonService: ApPersonService, - private utilityService: UtilityService, private apAudienceService: ApAudienceService, private apMentionService: ApMentionService, @@ -89,10 +91,17 @@ export class ApNoteService { private apDbResolverService: ApDbResolverService, private apLoggerService: ApLoggerService, private readonly apUtilityService: ApUtilityService, + private readonly customEmojiService: CustomEmojiService, + private readonly timeService: TimeService, ) { this.logger = this.apLoggerService.logger; } + @bindThis + public onModuleInit() { + this.apPersonService = this.moduleRef.get('ApPersonService'); + } + @bindThis public validateNote( object: IObject, @@ -198,7 +207,7 @@ export class ApNoteService { const uri = getOneApId(note.attributedTo); // ローカルで投稿者を検索し、もし凍結されていたらスキップ - // eslint-disable-next-line no-param-reassign + actor ??= await this.apPersonService.fetchPerson(uri) as MiRemoteUser | undefined; if (actor && actor.isSuspended) { throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to create note ${entryUri}: actor ${uri} has been suspended`); @@ -230,7 +239,6 @@ export class ApNoteService { } //#endregion - // eslint-disable-next-line no-param-reassign actor ??= await this.apPersonService.resolvePerson(uri, resolver) as MiRemoteUser; // 解決した投稿者が凍結されていたらスキップ @@ -285,7 +293,7 @@ export class ApNoteService { const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id }); const tryCreateVote = async (name: string, index: number): Promise => { - if (poll.expiresAt && Date.now() > new Date(poll.expiresAt).getTime()) { + if (poll.expiresAt && this.timeService.now > new Date(poll.expiresAt).getTime()) { this.logger.warn(`vote to expired poll from AP: actor=${actor.username}@${actor.host}, note=${note.id}, choice=${name}`); } else if (index >= 0) { this.logger.info(`vote from AP: actor=${actor.username}@${actor.host}, note=${note.id}, choice=${name}`); @@ -464,7 +472,7 @@ export class ApNoteService { const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id }); const tryCreateVote = async (name: string, index: number): Promise => { - if (poll.expiresAt && Date.now() > new Date(poll.expiresAt).getTime()) { + if (poll.expiresAt && this.timeService.now > new Date(poll.expiresAt).getTime()) { this.logger.warn(`vote to expired poll from AP: actor=${actor.username}@${actor.host}, note=${note.id}, choice=${name}`); } else if (index >= 0) { this.logger.info(`vote from AP: actor=${actor.username}@${actor.host}, note=${note.id}, choice=${name}`); @@ -554,7 +562,7 @@ export class ApNoteService { const createFrom = haveSameAuthority ? value : uri; return await this.createNote(createFrom, undefined, options.resolver, true); } finally { - unlock(); + await unlock(); } } @@ -563,58 +571,60 @@ export class ApNoteService { // eslint-disable-next-line no-param-reassign host = this.utilityService.toPuny(host); - const eomjiTags = toArray(tags).filter(isEmoji); + const eomjiTags: (IApEmoji & { name: string })[] = toArray(tags) + .filter(tag => isEmoji(tag)) + .map(tag => ({ + ...tag, + name: tag.name.replaceAll(':', ''), + })) + .filter(tag => isValidEmojiName(tag.name)); - const existingEmojis = await this.emojisRepository.findBy({ - host, - name: In(eomjiTags.map(tag => tag.name.replaceAll(':', ''))), - }); + const emojiKeys = eomjiTags.map(tag => encodeEmojiKey({ name: tag.name, host })); + const existingEmojis = await this.customEmojiService.emojisByKeyCache.fetchMany(emojiKeys); return await Promise.all(eomjiTags.map(async tag => { - const name = tag.name.replaceAll(':', ''); + const name = tag.name; tag.icon = toSingle(tag.icon); - const exists = existingEmojis.find(x => x.name === name); + const exists = existingEmojis.values.find(x => x.name === name); if (exists) { if ((exists.updatedAt == null) - || (tag.id != null && exists.uri == null) - || (new Date(tag.updated) > exists.updatedAt) + || (tag.id != null && exists.uri == null) // TODO should we check for ID changes? + || (new Date(tag.updated) > exists.updatedAt) // TODO make sure tag.updated actually exists || (tag.icon.url !== exists.originalUrl) + // TODO check for license changes + // TODO check for sensitive changes ) { - await this.emojisRepository.update({ + return await this.customEmojiService.updateEmoji({ host, name, }, { uri: tag.id, originalUrl: tag.icon.url, publicUrl: tag.icon.url, - updatedAt: new Date(), + updatedAt: this.timeService.date, // _misskey_license が存在しなければ `null` license: (tag._misskey_license?.freeText ?? null), }); - - const emoji = await this.emojisRepository.findOneBy({ host, name }); - if (emoji == null) throw new Error(`emoji update failed: ${name}:${host}`); - return emoji; } return exists; } - this.logger.info(`register emoji host=${host}, name=${name}`); - - return await this.emojisRepository.insertOne({ + return await this.customEmojiService.createEmoji({ id: this.idService.gen(), host, name, uri: tag.id, originalUrl: tag.icon.url, publicUrl: tag.icon.url, - updatedAt: new Date(), + updatedAt: this.timeService.date, aliases: [], + localOnly: false, + isSensitive: tag.sensitive === true, // _misskey_license が存在しなければ `null` - license: (tag._misskey_license?.freeText ?? null) + license: (tag._misskey_license?.freeText ?? null), }); })); } diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 9d7b7c5074..25cd0ce8c0 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -3,23 +3,25 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import promiseLimit from 'promise-limit'; -import { DataSource } from 'typeorm'; +import { DataSource, In } from 'typeorm'; import { ModuleRef } from '@nestjs/core'; import { UnrecoverableError } from 'bullmq'; import { DI } from '@/di-symbols.js'; import type { FollowingsRepository, InstancesRepository, MiMeta, UserProfilesRepository, UserPublickeysRepository, UsersRepository } from '@/models/_.js'; import type { Config } from '@/config.js'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; +import { isRemoteUser, isLocalUser } from '@/models/User.js'; import { MiUser } from '@/models/User.js'; import { truncate } from '@/misc/truncate.js'; import type { CacheService } from '@/core/CacheService.js'; +import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; import type Logger from '@/logger.js'; import type { MiNote } from '@/models/Note.js'; -import type { IdService } from '@/core/IdService.js'; +import { IdService } from '@/core/IdService.js'; import type { MfmService } from '@/core/MfmService.js'; import { toArray } from '@/misc/prelude/array.js'; import type { GlobalEventService } from '@/core/GlobalEventService.js'; @@ -31,27 +33,26 @@ import type UsersChart from '@/core/chart/charts/users.js'; import type InstanceChart from '@/core/chart/charts/instance.js'; import type { HashtagService } from '@/core/HashtagService.js'; import { MiUserNotePining } from '@/models/UserNotePining.js'; -import type { UtilityService } from '@/core/UtilityService.js'; -import type { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; -import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; +import type { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import type { AccountMoveService } from '@/core/AccountMoveService.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { AppLockService } from '@/core/AppLockService.js'; -import { MemoryKVCache } from '@/misc/cache.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { TimeService } from '@/global/TimeService.js'; import { verifyFieldLinks } from '@/misc/verify-field-link.js'; import { isRetryableError } from '@/misc/is-retryable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { getApId, getApType, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js'; +import { ApLoggerService } from '../ApLoggerService.js'; import { extractApHashtags } from './tag.js'; import type { OnModuleInit } from '@nestjs/common'; import type { ApNoteService } from './ApNoteService.js'; import type { ApMfmService } from '../ApMfmService.js'; import type { ApResolverService, Resolver } from '../ApResolverService.js'; -import type { ApLoggerService } from '../ApLoggerService.js'; import type { ApImageService } from './ApImageService.js'; import type { IActor, ICollection, IObject, IOrderedCollection } from '../type.js'; @@ -61,15 +62,15 @@ const nameLength = 128; type Field = Record<'name' | 'value', string>; @Injectable() -export class ApPersonService implements OnModuleInit, OnApplicationShutdown { - // Moved from ApDbResolverService - private readonly publicKeyByKeyIdCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h - private readonly publicKeyByUserIdCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h +export class ApPersonService implements OnModuleInit { + // Moved from CacheService + public readonly uriPersonCache: ManagedQuantumKVCache; + + // Moved from ApDbResolverService + private readonly publicKeyByKeyIdCache: ManagedQuantumKVCache; + private readonly publicKeyByUserIdCache: ManagedQuantumKVCache; - private utilityService: UtilityService; - private userEntityService: UserEntityService; private driveFileEntityService: DriveFileEntityService; - private idService: IdService; private globalEventService: GlobalEventService; private federatedInstanceService: FederatedInstanceService; private fetchInstanceMetadataService: FetchInstanceMetadataService; @@ -82,7 +83,6 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { private hashtagService: HashtagService; private usersChart: UsersChart; private instanceChart: InstanceChart; - private apLoggerService: ApLoggerService; private accountMoveService: AccountMoveService; private logger: Logger; @@ -114,17 +114,71 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { private followingsRepository: FollowingsRepository, private roleService: RoleService, - private readonly apUtilityService: ApUtilityService, private readonly httpRequestService: HttpRequestService, private readonly appLockService: AppLockService, + private readonly cacheManagementService: CacheManagementService, + private readonly utilityService: UtilityService, + private readonly apUtilityService: ApUtilityService, + private readonly idService: IdService, + private readonly timeService: TimeService, + + apLoggerService: ApLoggerService, ) { + this.logger = apLoggerService.logger; + + this.uriPersonCache = this.cacheManagementService.createQuantumKVCache('uriPerson', { + lifetime: 1000 * 60 * 30, // 30m + fetcher: async (uri) => { + const { id } = await this.usersRepository + .createQueryBuilder('user') + .select('user.id') + .where({ uri }) + .getOneOrFail() as { id: string }; + return id; + }, + optionalFetcher: async (uri) => { + const res = await this.usersRepository + .createQueryBuilder('user') + .select('user.id') + .where({ uri }) + .getOne() as { id: string } | null; + return res?.id; + }, + bulkFetcher: async (uris) => { + const users = await this.usersRepository + .createQueryBuilder('user') + .select('user.id') + .addSelect('user.uri') + .where({ uri: In(uris) }) + .getMany() as { id: string, uri: string }[]; + return users.map(user => [user.uri, user.id]); + }, + }); + + this.publicKeyByKeyIdCache = this.cacheManagementService.createQuantumKVCache('publicKeyByKeyId', { + lifetime: 1000 * 60 * 60 * 12, // 12h + fetcher: async (keyId) => await this.userPublickeysRepository.findOneByOrFail({ keyId }), + optionalFetcher: async (keyId) => await this.userPublickeysRepository.findOneBy({ keyId }), + bulkFetcher: async (keyIds) => { + const publicKeys = await this.userPublickeysRepository.findBy({ keyId: In(keyIds) }); + return publicKeys.map(k => [k.keyId, k]); + }, + }); + + this.publicKeyByUserIdCache = this.cacheManagementService.createQuantumKVCache('publicKeyByUserId', { + lifetime: 1000 * 60 * 60 * 12, // 12h + fetcher: async (userId) => await this.userPublickeysRepository.findOneByOrFail({ userId }), + optionalFetcher: async (userId) => await this.userPublickeysRepository.findOneBy({ userId }), + bulkFetcher: async (userIds) => { + const publicKeys = await this.userPublickeysRepository.findBy({ userId: In(userIds) }); + return publicKeys.map(k => [k.userId, k]); + }, + }); } + @bindThis onModuleInit(): void { - this.utilityService = this.moduleRef.get('UtilityService'); - this.userEntityService = this.moduleRef.get('UserEntityService'); this.driveFileEntityService = this.moduleRef.get('DriveFileEntityService'); - this.idService = this.moduleRef.get('IdService'); this.globalEventService = this.moduleRef.get('GlobalEventService'); this.federatedInstanceService = this.moduleRef.get('FederatedInstanceService'); this.fetchInstanceMetadataService = this.moduleRef.get('FetchInstanceMetadataService'); @@ -137,13 +191,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { this.hashtagService = this.moduleRef.get('HashtagService'); this.usersChart = this.moduleRef.get('UsersChart'); this.instanceChart = this.moduleRef.get('InstanceChart'); - this.apLoggerService = this.moduleRef.get('ApLoggerService'); this.accountMoveService = this.moduleRef.get('AccountMoveService'); - this.logger = this.apLoggerService.logger; - } - - onApplicationShutdown(): void { - this.dispose(); } /** @@ -246,14 +294,14 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { */ @bindThis public async fetchPerson(uri: string): Promise { - const cached = this.cacheService.uriPersonCache.get(uri) as MiLocalUser | MiRemoteUser | null | undefined; - if (cached) return cached; + const cached = await this.uriPersonCache.fetchMaybe(uri); + if (cached) return await this.cacheService.findOptionalUserById(cached) as MiRemoteUser | MiLocalUser | undefined ?? null; // URIがこのサーバーを指しているならデータベースからフェッチ if (uri.startsWith(`${this.config.url}/`)) { const id = uri.split('/').pop(); const u = await this.usersRepository.findOneBy({ id }) as MiLocalUser | null; - if (u) this.cacheService.uriPersonCache.set(uri, u); + if (u) await this.uriPersonCache.set(uri, u.id); return u; } @@ -261,7 +309,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const exist = await this.usersRepository.findOneBy({ uri }) as MiLocalUser | MiRemoteUser | null; if (exist) { - this.cacheService.uriPersonCache.set(uri, exist); + await this.uriPersonCache.set(uri, exist.id); return exist; } //#endregion @@ -415,13 +463,13 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { avatarId: null, bannerId: null, backgroundId: null, - lastFetchedAt: new Date(), + lastFetchedAt: this.timeService.date, name: truncate(person.name, nameLength), noindex: (person as any).noindex ?? false, enableRss: person.enableRss === true, isLocked: person.manuallyApprovesFollowers, movedToUri: person.movedTo, - movedAt: person.movedTo ? new Date() : null, + movedAt: person.movedTo ? this.timeService.date : null, alsoKnownAs: person.alsoKnownAs, // We use "!== false" to handle incorrect types, missing / null values, and "default to true" logic. hideOnlineStatus: person.hideOnlineStatus !== false, @@ -502,12 +550,15 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { if (user == null) throw new Error(`failed to create user - user is null: ${uri}`); // Register to the cache - this.cacheService.uriPersonCache.set(user.uri, user); + await this.uriPersonCache.set(user.uri, user.id); // Register public key to the cache. - // Value may be null, which indicates that the user has no defined key. (optimization) - this.publicKeyByUserIdCache.set(user.id, publicKey); - if (publicKey) this.publicKeyByKeyIdCache.set(publicKey.keyId, publicKey); + if (publicKey) { + await Promise.all([ + this.publicKeyByKeyIdCache.set(publicKey.keyId, publicKey), + this.publicKeyByUserIdCache.set(publicKey.userId, publicKey), + ]); + } // Register host if (this.meta.enableStatsForFederatedInstances) { @@ -532,7 +583,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { user = { ...user, ...updates }; // Register to the cache - this.cacheService.uriPersonCache.set(user.uri, user); + await this.uriPersonCache.set(user.uri, user.id); } catch (err) { // Permanent error implies hidden or inaccessible, which is a normal thing. if (isRetryableError(err)) { @@ -627,7 +678,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const verifiedLinks = await verifyFieldLinks(fields, profileUrls, this.httpRequestService); const updates = { - lastFetchedAt: new Date(), + lastFetchedAt: this.timeService.date, inbox: person.inbox, sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null, followersUri: person.followers ? getApId(person.followers) : undefined, @@ -681,7 +732,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { return false; })(); - if (moving) updates.movedAt = new Date(); + if (moving) updates.movedAt = this.timeService.date; // Update user if (!(await this.usersRepository.update({ id: exist.id, isDeleted: false }, updates)).affected) { @@ -698,18 +749,21 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { // Create or update key await this.userPublickeysRepository.save(publicKey); - this.publicKeyByKeyIdCache.set(person.publicKey.id, publicKey); - this.publicKeyByUserIdCache.set(exist.id, publicKey); + // Save it to the cache + await Promise.all([ + this.publicKeyByKeyIdCache.set(publicKey.keyId, publicKey), + this.publicKeyByUserIdCache.set(publicKey.userId, publicKey), + ]); } else { const existingPublicKey = await this.userPublickeysRepository.findOneBy({ userId: exist.id }); if (existingPublicKey) { // Delete key - await this.userPublickeysRepository.delete({ userId: exist.id }); - this.publicKeyByKeyIdCache.delete(existingPublicKey.keyId); + await Promise.all([ + this.userPublickeysRepository.delete({ userId: existingPublicKey.userId }), + this.publicKeyByUserIdCache.delete(existingPublicKey.userId), + this.publicKeyByKeyIdCache.delete(existingPublicKey.keyId), + ]); } - - // Null indicates that the user has no key. (optimization) - this.publicKeyByUserIdCache.set(exist.id, null); } let _description: string | null = null; @@ -760,8 +814,6 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const updated = { ...exist, ...updates }; - this.cacheService.uriPersonCache.set(uri, updated); - // 移行処理を行う if (updated.movedAt && ( // 初めて移行する場合はmovedAtがnullなので移行処理を許可 @@ -817,7 +869,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const createFrom = haveSameAuthority ? value : uri; return await this._createPerson(createFrom, resolver); } finally { - unlock(); + await unlock(); } } @@ -841,7 +893,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { @bindThis public async updateFeatured(userId: MiUser['id'], resolver?: Resolver): Promise { const user = await this.usersRepository.findOneByOrFail({ id: userId, isDeleted: false }); - if (!this.userEntityService.isRemoteUser(user)) return; + if (!isRemoteUser(user)) return; if (!user.featured) return; this.logger.info(`Updating the featured: ${user.uri}`); @@ -884,7 +936,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { for (const note of featuredNotes.filter(x => x != null)) { td -= 1000; transactionalEntityManager.insert(MiUserNotePining, { - id: this.idService.gen(Date.now() + td), + id: this.idService.gen(this.timeService.now + td), userId: user.id, noteId: note.id, }); @@ -906,7 +958,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { // まずサーバー内で検索して様子見 let dst = await this.fetchPerson(src.movedToUri); - if (dst && this.userEntityService.isLocalUser(dst)) { + if (dst && isLocalUser(dst)) { // targetがローカルユーザーだった場合データベースから引っ張ってくる dst = await this.usersRepository.findOneByOrFail({ uri: src.movedToUri }) as MiLocalUser; } else if (dst) { @@ -942,10 +994,10 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { } @bindThis - private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver, sentFrom: string): Promise { + private async isPublicCollection(collection: string | IObject | undefined, resolver: Resolver, sentFrom: string): Promise { if (collection) { - const resolved = await resolver.resolveCollection(collection, true, sentFrom).catch(() => null); - if (resolved) { + const resolved = await resolver.resolveCollection(collection, true, sentFrom); + { if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) { return true; } @@ -957,35 +1009,11 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { @bindThis public async findPublicKeyByUserId(userId: string): Promise { - const publicKey = this.publicKeyByUserIdCache.get(userId) ?? await this.userPublickeysRepository.findOneBy({ userId }); - - // This can technically keep a key cached "forever" if it's used enough, but that's ok. - // We can never have stale data because the publicKey caches are coherent. (cache updates whenever data changes) - if (publicKey) { - this.publicKeyByUserIdCache.set(publicKey.userId, publicKey); - this.publicKeyByKeyIdCache.set(publicKey.keyId, publicKey); - } - - return publicKey; + return await this.publicKeyByUserIdCache.fetchMaybe(userId) ?? null; } @bindThis public async findPublicKeyByKeyId(keyId: string): Promise { - const publicKey = this.publicKeyByKeyIdCache.get(keyId) ?? await this.userPublickeysRepository.findOneBy({ keyId }); - - // This can technically keep a key cached "forever" if it's used enough, but that's ok. - // We can never have stale data because the publicKey caches are coherent. (cache updates whenever data changes) - if (publicKey) { - this.publicKeyByUserIdCache.set(publicKey.userId, publicKey); - this.publicKeyByKeyIdCache.set(publicKey.keyId, publicKey); - } - - return publicKey; - } - - @bindThis - public dispose(): void { - this.publicKeyByUserIdCache.dispose(); - this.publicKeyByKeyIdCache.dispose(); + return await this.publicKeyByKeyIdCache.fetchMaybe(keyId) ?? null; } } diff --git a/packages/backend/src/core/chart/ChartManagementService.ts b/packages/backend/src/core/chart/ChartManagementService.ts index 4f151ff73d..95bcb22cc4 100644 --- a/packages/backend/src/core/chart/ChartManagementService.ts +++ b/packages/backend/src/core/chart/ChartManagementService.ts @@ -7,7 +7,9 @@ import { Injectable } from '@nestjs/common'; import { bindThis } from '@/decorators.js'; import { ChartLoggerService } from '@/core/chart/ChartLoggerService.js'; +import { TimeService, type TimerHandle } from '@/global/TimeService.js'; import Logger from '@/logger.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import FederationChart from './charts/federation.js'; import NotesChart from './charts/notes.js'; import UsersChart from './charts/users.js'; @@ -25,7 +27,7 @@ import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() export class ChartManagementService implements OnApplicationShutdown { private charts; - private saveIntervalId: NodeJS.Timeout; + private saveIntervalId: TimerHandle; private readonly logger: Logger; constructor( @@ -41,6 +43,8 @@ export class ChartManagementService implements OnApplicationShutdown { private perUserFollowingChart: PerUserFollowingChart, private perUserDriveChart: PerUserDriveChart, private apRequestChart: ApRequestChart, + private readonly timeService: TimeService, + chartLoggerService: ChartLoggerService, ) { this.charts = [ @@ -63,21 +67,23 @@ export class ChartManagementService implements OnApplicationShutdown { @bindThis public async start() { // 20分おきにメモリ情報をDBに書き込み - this.saveIntervalId = setInterval(async () => { + this.saveIntervalId = this.timeService.startTimer(async () => { for (const chart of this.charts) { await chart.save(); } this.logger.info('All charts saved'); - }, 1000 * 60 * 20); + }, 1000 * 60 * 20, { repeated: true }); } @bindThis public async dispose(): Promise { - clearInterval(this.saveIntervalId); + this.timeService.stopTimer(this.saveIntervalId); if (process.env.NODE_ENV !== 'test') { this.logger.info('Saving charts for shutdown...'); for (const chart of this.charts) { - await chart.save(); + await chart.save().catch(err => { + this.logger.error(`Error saving chart: ${renderInlineError(err)}`); + }); } this.logger.info('All charts saved'); } diff --git a/packages/backend/src/core/chart/charts/active-users.ts b/packages/backend/src/core/chart/charts/active-users.ts index 05905f3782..20432fb293 100644 --- a/packages/backend/src/core/chart/charts/active-users.ts +++ b/packages/backend/src/core/chart/charts/active-users.ts @@ -10,6 +10,7 @@ import type { MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import Chart from '../core.js'; import { ChartLoggerService } from '../ChartLoggerService.js'; import { name, schema } from './entities/active-users.js'; @@ -31,10 +32,15 @@ export default class ActiveUsersChart extends Chart { // eslint-d private appLockService: AppLockService, private chartLoggerService: ChartLoggerService, private idService: IdService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(): Promise>> { return {}; } @@ -48,12 +54,12 @@ export default class ActiveUsersChart extends Chart { // eslint-d const createdAt = this.idService.parse(user.id).date; await this.commit({ 'read': [user.id], - 'registeredWithinWeek': (Date.now() - createdAt.getTime() < week) ? [user.id] : [], - 'registeredWithinMonth': (Date.now() - createdAt.getTime() < month) ? [user.id] : [], - 'registeredWithinYear': (Date.now() - createdAt.getTime() < year) ? [user.id] : [], - 'registeredOutsideWeek': (Date.now() - createdAt.getTime() > week) ? [user.id] : [], - 'registeredOutsideMonth': (Date.now() - createdAt.getTime() > month) ? [user.id] : [], - 'registeredOutsideYear': (Date.now() - createdAt.getTime() > year) ? [user.id] : [], + 'registeredWithinWeek': (this.timeService.now - createdAt.getTime() < week) ? [user.id] : [], + 'registeredWithinMonth': (this.timeService.now - createdAt.getTime() < month) ? [user.id] : [], + 'registeredWithinYear': (this.timeService.now - createdAt.getTime() < year) ? [user.id] : [], + 'registeredOutsideWeek': (this.timeService.now - createdAt.getTime() > week) ? [user.id] : [], + 'registeredOutsideMonth': (this.timeService.now - createdAt.getTime() > month) ? [user.id] : [], + 'registeredOutsideYear': (this.timeService.now - createdAt.getTime() > year) ? [user.id] : [], }); } diff --git a/packages/backend/src/core/chart/charts/ap-request.ts b/packages/backend/src/core/chart/charts/ap-request.ts index 04e771a95b..8cae5753c7 100644 --- a/packages/backend/src/core/chart/charts/ap-request.ts +++ b/packages/backend/src/core/chart/charts/ap-request.ts @@ -6,6 +6,7 @@ import { Injectable, Inject } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import Chart from '../core.js'; @@ -24,10 +25,15 @@ export default class ApRequestChart extends Chart { // eslint-dis private appLockService: AppLockService, private chartLoggerService: ChartLoggerService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(): Promise>> { return {}; } diff --git a/packages/backend/src/core/chart/charts/drive.ts b/packages/backend/src/core/chart/charts/drive.ts index 613e074a9f..cce07f3b5b 100644 --- a/packages/backend/src/core/chart/charts/drive.ts +++ b/packages/backend/src/core/chart/charts/drive.ts @@ -7,6 +7,7 @@ import { Injectable, Inject } from '@nestjs/common'; import { DataSource } from 'typeorm'; import type { MiDriveFile } from '@/models/DriveFile.js'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import Chart from '../core.js'; @@ -25,10 +26,15 @@ export default class DriveChart extends Chart { // eslint-disable private appLockService: AppLockService, private chartLoggerService: ChartLoggerService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(): Promise>> { return {}; } diff --git a/packages/backend/src/core/chart/charts/federation.ts b/packages/backend/src/core/chart/charts/federation.ts index 4bbb5437cc..199c263cce 100644 --- a/packages/backend/src/core/chart/charts/federation.ts +++ b/packages/backend/src/core/chart/charts/federation.ts @@ -7,6 +7,7 @@ import { Injectable, Inject } from '@nestjs/common'; import { DataSource } from 'typeorm'; import type { FollowingsRepository, InstancesRepository, MiMeta } from '@/models/_.js'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import Chart from '../core.js'; @@ -34,10 +35,15 @@ export default class FederationChart extends Chart { // eslint-di private appLockService: AppLockService, private chartLoggerService: ChartLoggerService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(): Promise>> { return { }; diff --git a/packages/backend/src/core/chart/charts/instance.ts b/packages/backend/src/core/chart/charts/instance.ts index 97f3bc6f2b..ca6c1c5026 100644 --- a/packages/backend/src/core/chart/charts/instance.ts +++ b/packages/backend/src/core/chart/charts/instance.ts @@ -9,6 +9,7 @@ import type { DriveFilesRepository, FollowingsRepository, UsersRepository, Notes import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiNote } from '@/models/Note.js'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; @@ -41,10 +42,15 @@ export default class InstanceChart extends Chart { // eslint-disa private utilityService: UtilityService, private appLockService: AppLockService, private chartLoggerService: ChartLoggerService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema, true); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(group: string): Promise>> { const [ notesCount, diff --git a/packages/backend/src/core/chart/charts/notes.ts b/packages/backend/src/core/chart/charts/notes.ts index f763b5fffa..43cabd0b98 100644 --- a/packages/backend/src/core/chart/charts/notes.ts +++ b/packages/backend/src/core/chart/charts/notes.ts @@ -8,6 +8,7 @@ import { Not, IsNull, DataSource } from 'typeorm'; import type { NotesRepository } from '@/models/_.js'; import type { MiNote } from '@/models/Note.js'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import Chart from '../core.js'; @@ -29,10 +30,15 @@ export default class NotesChart extends Chart { // eslint-disable private appLockService: AppLockService, private chartLoggerService: ChartLoggerService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(): Promise>> { const [localCount, remoteCount] = await Promise.all([ this.notesRepository.countBy({ userHost: IsNull() }), diff --git a/packages/backend/src/core/chart/charts/per-user-drive.ts b/packages/backend/src/core/chart/charts/per-user-drive.ts index 404964d8b7..663abc5f00 100644 --- a/packages/backend/src/core/chart/charts/per-user-drive.ts +++ b/packages/backend/src/core/chart/charts/per-user-drive.ts @@ -8,6 +8,7 @@ import { DataSource } from 'typeorm'; import type { DriveFilesRepository } from '@/models/_.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import { bindThis } from '@/decorators.js'; @@ -31,10 +32,15 @@ export default class PerUserDriveChart extends Chart { // eslint- private appLockService: AppLockService, private driveFileEntityService: DriveFileEntityService, private chartLoggerService: ChartLoggerService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema, true); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(group: string): Promise>> { const [count, size] = await Promise.all([ this.driveFilesRepository.countBy({ userId: group }), diff --git a/packages/backend/src/core/chart/charts/per-user-following.ts b/packages/backend/src/core/chart/charts/per-user-following.ts index 8d75a30e9a..71678b0573 100644 --- a/packages/backend/src/core/chart/charts/per-user-following.ts +++ b/packages/backend/src/core/chart/charts/per-user-following.ts @@ -7,6 +7,8 @@ import { Injectable, Inject } from '@nestjs/common'; import { Not, IsNull, DataSource } from 'typeorm'; import type { MiUser } from '@/models/User.js'; import { AppLockService } from '@/core/AppLockService.js'; +import { CacheService } from '@/core/CacheService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import type { FollowingsRepository } from '@/models/_.js'; @@ -15,7 +17,6 @@ import Chart from '../core.js'; import { ChartLoggerService } from '../ChartLoggerService.js'; import { name, schema } from './entities/per-user-following.js'; import type { KVs } from '../core.js'; -import { CacheService } from '@/core/CacheService.js'; /** * ユーザーごとのフォローに関するチャート @@ -33,10 +34,15 @@ export default class PerUserFollowingChart extends Chart { // esl private userEntityService: UserEntityService, private chartLoggerService: ChartLoggerService, private readonly cacheService: CacheService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema, true); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(group: string): Promise>> { const [ followees, diff --git a/packages/backend/src/core/chart/charts/per-user-notes.ts b/packages/backend/src/core/chart/charts/per-user-notes.ts index e4900772bb..1182fa2984 100644 --- a/packages/backend/src/core/chart/charts/per-user-notes.ts +++ b/packages/backend/src/core/chart/charts/per-user-notes.ts @@ -8,6 +8,7 @@ import { DataSource } from 'typeorm'; import type { MiUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import type { NotesRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; @@ -30,10 +31,15 @@ export default class PerUserNotesChart extends Chart { // eslint- private appLockService: AppLockService, private chartLoggerService: ChartLoggerService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema, true); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(group: string): Promise>> { const [count] = await Promise.all([ this.notesRepository.countBy({ userId: group }), diff --git a/packages/backend/src/core/chart/charts/per-user-pv.ts b/packages/backend/src/core/chart/charts/per-user-pv.ts index 31708fefa8..75a61aae07 100644 --- a/packages/backend/src/core/chart/charts/per-user-pv.ts +++ b/packages/backend/src/core/chart/charts/per-user-pv.ts @@ -7,6 +7,7 @@ import { Injectable, Inject } from '@nestjs/common'; import { DataSource } from 'typeorm'; import type { MiUser } from '@/models/User.js'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import Chart from '../core.js'; @@ -25,10 +26,15 @@ export default class PerUserPvChart extends Chart { // eslint-dis private appLockService: AppLockService, private chartLoggerService: ChartLoggerService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema, true); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(): Promise>> { return {}; } diff --git a/packages/backend/src/core/chart/charts/per-user-reactions.ts b/packages/backend/src/core/chart/charts/per-user-reactions.ts index c29c4d2870..9fb78a28e9 100644 --- a/packages/backend/src/core/chart/charts/per-user-reactions.ts +++ b/packages/backend/src/core/chart/charts/per-user-reactions.ts @@ -8,6 +8,7 @@ import { DataSource } from 'typeorm'; import type { MiUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; @@ -28,10 +29,15 @@ export default class PerUserReactionsChart extends Chart { // esl private appLockService: AppLockService, private userEntityService: UserEntityService, private chartLoggerService: ChartLoggerService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema, true); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(group: string): Promise>> { return {}; } diff --git a/packages/backend/src/core/chart/charts/test-grouped.ts b/packages/backend/src/core/chart/charts/test-grouped.ts index 7a2844f4ed..6cc48d483d 100644 --- a/packages/backend/src/core/chart/charts/test-grouped.ts +++ b/packages/backend/src/core/chart/charts/test-grouped.ts @@ -6,6 +6,7 @@ import { Injectable, Inject } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; @@ -25,11 +26,17 @@ export default class TestGroupedChart extends Chart { // eslint-d private db: DataSource, private appLockService: AppLockService, + private readonly timeService: TimeService, + logger: Logger, ) { super(db, (k) => appLockService.getChartInsertLock(k), logger, name, schema, true); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(group: string): Promise>> { return { 'foo.total': this.total[group], diff --git a/packages/backend/src/core/chart/charts/test-intersection.ts b/packages/backend/src/core/chart/charts/test-intersection.ts index b8d0556c9f..d0ae1dab24 100644 --- a/packages/backend/src/core/chart/charts/test-intersection.ts +++ b/packages/backend/src/core/chart/charts/test-intersection.ts @@ -6,6 +6,7 @@ import { Injectable, Inject } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; @@ -23,11 +24,17 @@ export default class TestIntersectionChart extends Chart { // esl private db: DataSource, private appLockService: AppLockService, + private readonly timeService: TimeService, + logger: Logger, ) { super(db, (k) => appLockService.getChartInsertLock(k), logger, name, schema); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(): Promise>> { return {}; } diff --git a/packages/backend/src/core/chart/charts/test-unique.ts b/packages/backend/src/core/chart/charts/test-unique.ts index f94e008059..54a081fe2a 100644 --- a/packages/backend/src/core/chart/charts/test-unique.ts +++ b/packages/backend/src/core/chart/charts/test-unique.ts @@ -6,6 +6,7 @@ import { Injectable, Inject } from '@nestjs/common'; import { DataSource } from 'typeorm'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; @@ -23,11 +24,17 @@ export default class TestUniqueChart extends Chart { // eslint-di private db: DataSource, private appLockService: AppLockService, + private readonly timeService: TimeService, + logger: Logger, ) { super(db, (k) => appLockService.getChartInsertLock(k), logger, name, schema); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(): Promise>> { return {}; } diff --git a/packages/backend/src/core/chart/charts/test.ts b/packages/backend/src/core/chart/charts/test.ts index a90dc8f99b..e95259f3b2 100644 --- a/packages/backend/src/core/chart/charts/test.ts +++ b/packages/backend/src/core/chart/charts/test.ts @@ -9,6 +9,7 @@ import { AppLockService } from '@/core/AppLockService.js'; import { DI } from '@/di-symbols.js'; import Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; +import { TimeService } from '@/global/TimeService.js'; import Chart from '../core.js'; import { name, schema } from './entities/test.js'; import type { KVs } from '../core.js'; @@ -25,11 +26,17 @@ export default class TestChart extends Chart { // eslint-disable- private db: DataSource, private appLockService: AppLockService, + private readonly timeService: TimeService, + logger: Logger, ) { super(db, (k) => appLockService.getChartInsertLock(k), logger, name, schema); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(): Promise>> { return { 'foo.total': this.total, diff --git a/packages/backend/src/core/chart/charts/users.ts b/packages/backend/src/core/chart/charts/users.ts index 840522ae9b..91bf972371 100644 --- a/packages/backend/src/core/chart/charts/users.ts +++ b/packages/backend/src/core/chart/charts/users.ts @@ -7,6 +7,7 @@ import { Injectable, Inject } from '@nestjs/common'; import { Not, IsNull, Like, DataSource } from 'typeorm'; import type { MiUser } from '@/models/User.js'; import { AppLockService } from '@/core/AppLockService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import type { UsersRepository } from '@/models/_.js'; @@ -31,10 +32,15 @@ export default class UsersChart extends Chart { // eslint-disable private appLockService: AppLockService, private userEntityService: UserEntityService, private chartLoggerService: ChartLoggerService, + private readonly timeService: TimeService, ) { super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema); } + protected getCurrentDate(): Date { + return this.timeService.date; + } + protected async tickMajor(): Promise>> { const [localCount, remoteCount] = await Promise.all([ // that Not(Like()) is ugly, but it matches the logic in diff --git a/packages/backend/src/core/chart/core.ts b/packages/backend/src/core/chart/core.ts index 234c1d63b4..2f1ed27132 100644 --- a/packages/backend/src/core/chart/core.ts +++ b/packages/backend/src/core/chart/core.ts @@ -16,6 +16,7 @@ import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { MiRepository, miRepository } from '@/models/_.js'; import type { DataSource, Repository } from 'typeorm'; +import type { Lock } from 'redis-lock'; const COLUMN_PREFIX = '___' as const; const UNIQUE_TEMP_COLUMN_PREFIX = 'unique_temp___' as const; @@ -202,9 +203,7 @@ export default abstract class Chart { return [y, m, d, h, _m, _s, _ms]; } - private static getCurrentDate() { - return Chart.parseDate(new Date()); - } + protected abstract getCurrentDate(): Date; public static schemaToEntity(name: string, schema: Schema, grouped = false): { hour: EntitySchema, @@ -260,11 +259,11 @@ export default abstract class Chart { }; } - private lock: (key: string) => Promise<() => void>; + private lock: Lock; constructor( db: DataSource, - lock: (key: string) => Promise<() => void>, + lock: Lock, logger: Logger, name: string, schema: T, @@ -324,7 +323,7 @@ export default abstract class Chart { */ @bindThis private async claimCurrentLog(group: string | null, span: 'hour' | 'day'): Promise> { - const [y, m, d, h] = Chart.getCurrentDate(); + const [y, m, d, h] = Chart.parseDate(this.getCurrentDate()); const current = dateUTC( span === 'hour' ? [y, m, d, h] : @@ -402,7 +401,7 @@ export default abstract class Chart { return log; } finally { - unlock(); + await unlock(); } } @@ -579,7 +578,7 @@ export default abstract class Chart { @bindThis public async clean(): Promise { - const current = dateUTC(Chart.getCurrentDate()); + const current = this.getCurrentDate(); // 一日以上前かつ三日以内 const gt = Chart.dateToTimestamp(current) - (60 * 60 * 24 * 3); @@ -615,7 +614,7 @@ export default abstract class Chart { @bindThis public async getChartRaw(span: 'hour' | 'day', amount: number, cursor: Date | null, group: string | null = null): Promise> { - const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.getCurrentDate(); + const [y, m, d, h, _m, _s, _ms] = cursor ? Chart.parseDate(subtractTime(addTime(cursor, 1, span), 1)) : Chart.parseDate(this.getCurrentDate()); const [y2, m2, d2, h2] = cursor ? Chart.parseDate(addTime(cursor, 1, span)) : [] as never; const lt = dateUTC([y, m, d, h, _m, _s, _ms]); diff --git a/packages/backend/src/core/entities/DriveFileEntityService.ts b/packages/backend/src/core/entities/DriveFileEntityService.ts index c485555f90..a172f81eed 100644 --- a/packages/backend/src/core/entities/DriveFileEntityService.ts +++ b/packages/backend/src/core/entities/DriveFileEntityService.ts @@ -3,8 +3,9 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { forwardRef, Inject, Injectable } from '@nestjs/common'; +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; import { In } from 'typeorm'; +import { ModuleRef } from '@nestjs/core'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository } from '@/models/_.js'; import type { Config } from '@/config.js'; @@ -19,8 +20,8 @@ import { isMimeImage } from '@/misc/is-mime-image.js'; import { IdService } from '@/core/IdService.js'; import { UtilityService } from '../UtilityService.js'; import { VideoProcessingService } from '../VideoProcessingService.js'; -import { UserEntityService } from './UserEntityService.js'; import { DriveFolderEntityService } from './DriveFolderEntityService.js'; +import type { UserEntityService } from './UserEntityService.js'; type PackOptions = { detail?: boolean, @@ -29,18 +30,18 @@ type PackOptions = { }; @Injectable() -export class DriveFileEntityService { +export class DriveFileEntityService implements OnModuleInit { + private userEntityService: UserEntityService; + constructor( + private readonly moduleRef: ModuleRef, + @Inject(DI.config) private config: Config, @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, - // 循環参照のため / for circular dependency - @Inject(forwardRef(() => UserEntityService)) - private userEntityService: UserEntityService, - private utilityService: UtilityService, private driveFolderEntityService: DriveFolderEntityService, private videoProcessingService: VideoProcessingService, @@ -48,6 +49,11 @@ export class DriveFileEntityService { ) { } + @bindThis + public onModuleInit() { + this.userEntityService = this.moduleRef.get('UserEntityService'); + } + @bindThis public validateFileName(name: string): boolean { return ( diff --git a/packages/backend/src/core/entities/EmojiEntityService.ts b/packages/backend/src/core/entities/EmojiEntityService.ts index 490d3f2511..5f03df554c 100644 --- a/packages/backend/src/core/entities/EmojiEntityService.ts +++ b/packages/backend/src/core/entities/EmojiEntityService.ts @@ -3,17 +3,23 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Inject, Injectable } from '@nestjs/common'; +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { ModuleRef } from '@nestjs/core'; import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { EmojisRepository, MiRole, RolesRepository } from '@/models/_.js'; import type { Packed } from '@/misc/json-schema.js'; import type { MiEmoji } from '@/models/Emoji.js'; import { bindThis } from '@/decorators.js'; +import type { CustomEmojiService } from '@/core/CustomEmojiService.js'; @Injectable() -export class EmojiEntityService { +export class EmojiEntityService implements OnModuleInit { + private customEmojiService: CustomEmojiService; + constructor( + private readonly moduleRef: ModuleRef, + @Inject(DI.emojisRepository) private emojisRepository: EmojisRepository, @Inject(DI.rolesRepository) @@ -21,11 +27,16 @@ export class EmojiEntityService { ) { } + @bindThis + public onModuleInit(): void { + this.customEmojiService = this.moduleRef.get('CustomEmojiService'); + } + @bindThis public async packSimple( src: MiEmoji['id'] | MiEmoji, ): Promise> { - const emoji = typeof src === 'object' ? src : await this.emojisRepository.findOneByOrFail({ id: src }); + const emoji = typeof src === 'object' ? src : await this.customEmojiService.emojisByIdCache.fetch(src); return { aliases: emoji.aliases, @@ -40,17 +51,24 @@ export class EmojiEntityService { } @bindThis - public packSimpleMany( - emojis: any[], + public async packSimpleMany( + emojis: readonly (MiEmoji | MiEmoji['id'])[], ) { - return Promise.all(emojis.map(x => this.packSimple(x))); + const toFetch = emojis.filter(emoji => typeof(emoji) === 'string'); + const fetched = new Map(await this.customEmojiService.emojisByIdCache.fetchMany(toFetch)); + return Promise.all(emojis.map(async x => { + if (typeof(x) === 'string') { + x = fetched.get(x) ?? await this.customEmojiService.emojisByIdCache.fetch(x); + } + return await this.packSimple(x); + })); } @bindThis public async packDetailed( src: MiEmoji['id'] | MiEmoji, ): Promise> { - const emoji = typeof src === 'object' ? src : await this.emojisRepository.findOneByOrFail({ id: src }); + const emoji = typeof src === 'object' ? src : await this.customEmojiService.emojisByIdCache.fetch(src); return { id: emoji.id, @@ -68,10 +86,17 @@ export class EmojiEntityService { } @bindThis - public packDetailedMany( - emojis: any[], + public async packDetailedMany( + emojis: readonly (MiEmoji | MiEmoji['id'])[], ): Promise[]> { - return Promise.all(emojis.map(x => this.packDetailed(x))); + const toFetch = emojis.filter(emoji => typeof(emoji) === 'string'); + const fetched = new Map(await this.customEmojiService.emojisByIdCache.fetchMany(toFetch)); + return Promise.all(emojis.map(async x => { + if (typeof(x) === 'string') { + x = fetched.get(x) ?? await this.customEmojiService.emojisByIdCache.fetch(x); + } + return this.packDetailed(x); + })); } @bindThis @@ -81,7 +106,7 @@ export class EmojiEntityService { roles?: Map }, ): Promise> { - const emoji = typeof src === 'object' ? src : await this.emojisRepository.findOneByOrFail({ id: src }); + const emoji = typeof src === 'object' ? src : await this.customEmojiService.emojisByIdCache.fetch(src); const roles = Array.of(); if (emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.length > 0) { @@ -136,7 +161,8 @@ export class EmojiEntityService { const emojiEntities = emojis.filter(x => typeof x === 'object') as MiEmoji[]; const emojiIdOnlyList = emojis.filter(x => typeof x === 'string') as string[]; if (emojiIdOnlyList.length > 0) { - emojiEntities.push(...await this.emojisRepository.findBy({ id: In(emojiIdOnlyList) })); + const fetched = await this.customEmojiService.emojisByIdCache.fetchMany(emojiIdOnlyList); + emojiEntities.push(...fetched.values); } // 特定ロール専用の絵文字である場合、そのロール情報をあらかじめまとめて取得しておく（pack側で都度取得も出来るが負荷が高いので） diff --git a/packages/backend/src/core/entities/FlashEntityService.ts b/packages/backend/src/core/entities/FlashEntityService.ts index 7b0150f5b6..c2575e69aa 100644 --- a/packages/backend/src/core/entities/FlashEntityService.ts +++ b/packages/backend/src/core/entities/FlashEntityService.ts @@ -40,7 +40,7 @@ export class FlashEntityService { // { schema: 'UserDetailed' } すると無限ループするので注意 const user = hint?.packedUser ?? await this.userEntityService.pack(flash.user ?? flash.userId, me); - let isLiked = undefined; + let isLiked: boolean | undefined = undefined; if (meId) { isLiked = hint?.likedFlashIds ? hint.likedFlashIds.includes(flash.id) diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index d7ab20db2c..c09374e897 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -14,6 +14,7 @@ import { SystemAccountService } from '@/core/SystemAccountService.js'; import type { Config } from '@/config.js'; import { DI } from '@/di-symbols.js'; import { DEFAULT_POLICIES } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class MetaEntityService { @@ -28,6 +29,7 @@ export class MetaEntityService { private adsRepository: AdsRepository, private systemAccountService: SystemAccountService, + private readonly timeService: TimeService, ) { } @bindThis @@ -39,18 +41,18 @@ export class MetaEntityService { } const ads = await this.adsRepository.createQueryBuilder('ads') - .where('ads.expiresAt > :now', { now: new Date() }) - .andWhere('ads.startsAt <= :now', { now: new Date() }) + .where('ads.expiresAt > :now', { now: this.timeService.date }) + .andWhere('ads.startsAt <= :now', { now: this.timeService.date }) .andWhere(new Brackets(qb => { // 曜日のビットフラグを確認する - qb.where('ads.dayOfWeek & :dayOfWeek > 0', { dayOfWeek: 1 << new Date().getDay() }) + qb.where('ads.dayOfWeek & :dayOfWeek > 0', { dayOfWeek: 1 << this.timeService.date.getDay() }) .orWhere('ads.dayOfWeek = 0'); })) .getMany(); // クライアントの手間を減らすためあらかじめJSONに変換しておく - let defaultLightTheme = null; - let defaultDarkTheme = null; + let defaultLightTheme: string | null = null; + let defaultDarkTheme: string | null = null; if (instance.defaultLightTheme) { try { defaultLightTheme = JSON.stringify(JSON5.parse(instance.defaultLightTheme)); diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 2b3defb189..373f05332f 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -14,9 +14,10 @@ import type { MiNote } from '@/models/Note.js'; import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta, MiPollVote, MiPoll, MiChannel, MiFollowing, NoteFavoritesRepository } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; import { DebounceLoader } from '@/misc/loader.js'; -import { IdService } from '@/core/IdService.js'; -import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; +import type { IdService } from '@/core/IdService.js'; +import type { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; import { QueryService } from '@/core/QueryService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { Config } from '@/config.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; import type { NoteVisibilityData } from '@/core/NoteVisibilityService.js'; @@ -106,6 +107,7 @@ export class NoteEntityService implements OnModuleInit { public readonly noteVisibilityService: NoteVisibilityService, private readonly queryService: QueryService, + private readonly timeService: TimeService, //private userEntityService: UserEntityService, //private driveFileEntityService: DriveFileEntityService, //private customEmojiService: CustomEmojiService, @@ -115,6 +117,7 @@ export class NoteEntityService implements OnModuleInit { ) { } + @bindThis onModuleInit() { this.userEntityService = this.moduleRef.get('UserEntityService'); this.driveFileEntityService = this.moduleRef.get('DriveFileEntityService'); @@ -133,7 +136,7 @@ export class NoteEntityService implements OnModuleInit { const followersOnlyBefore = packedNote.user.makeNotesFollowersOnlyBefore; if ((followersOnlyBefore != null) && ( - (followersOnlyBefore <= 0 && (Date.now() - new Date(packedNote.createdAt).getTime() > 0 - (followersOnlyBefore * 1000))) + (followersOnlyBefore <= 0 && (this.timeService.now - new Date(packedNote.createdAt).getTime() > 0 - (followersOnlyBefore * 1000))) || (followersOnlyBefore > 0 && (new Date(packedNote.createdAt).getTime() < followersOnlyBefore * 1000)) ) ) { @@ -387,7 +390,7 @@ export class NoteEntityService implements OnModuleInit { } // パフォーマンスのためノートが作成されてから2秒以上経っていない場合はリアクションを取得しない - if (this.idService.parse(note.id).date.getTime() + 2000 > Date.now()) { + if (this.idService.parse(note.id).date.getTime() + 2000 > this.timeService.now) { return undefined; } @@ -488,7 +491,7 @@ export class NoteEntityService implements OnModuleInit { @bindThis public async packAttachedFiles(fileIds: MiNote['fileIds'], packedFiles: Map | null>): Promise[]> { - const missingIds = []; + const missingIds: string[] = []; for (const id of fileIds) { if (!packedFiles.has(id)) missingIds.push(id); } diff --git a/packages/backend/src/core/entities/NoteReactionEntityService.ts b/packages/backend/src/core/entities/NoteReactionEntityService.ts index 46ec13704c..2b0d69b261 100644 --- a/packages/backend/src/core/entities/NoteReactionEntityService.ts +++ b/packages/backend/src/core/entities/NoteReactionEntityService.ts @@ -8,7 +8,7 @@ import { DI } from '@/di-symbols.js'; import type { NoteReactionsRepository } from '@/models/_.js'; import type { Packed } from '@/misc/json-schema.js'; import { bindThis } from '@/decorators.js'; -import { IdService } from '@/core/IdService.js'; +import type { IdService } from '@/core/IdService.js'; import type { OnModuleInit } from '@nestjs/common'; import type { } from '@/models/Blocking.js'; import type { MiUser } from '@/models/User.js'; @@ -38,6 +38,7 @@ export class NoteReactionEntityService implements OnModuleInit { ) { } + @bindThis onModuleInit() { this.userEntityService = this.moduleRef.get('UserEntityService'); this.noteEntityService = this.moduleRef.get('NoteEntityService'); diff --git a/packages/backend/src/core/entities/NotificationEntityService.ts b/packages/backend/src/core/entities/NotificationEntityService.ts index c00452110f..e30475b0f6 100644 --- a/packages/backend/src/core/entities/NotificationEntityService.ts +++ b/packages/backend/src/core/entities/NotificationEntityService.ts @@ -15,8 +15,8 @@ import type { Packed } from '@/misc/json-schema.js'; import { bindThis } from '@/decorators.js'; import { FilterUnionByProperty, groupedNotificationTypes } from '@/types.js'; import { CacheService } from '@/core/CacheService.js'; -import { RoleEntityService } from './RoleEntityService.js'; -import { ChatEntityService } from './ChatEntityService.js'; +import type { RoleEntityService } from './RoleEntityService.js'; +import type { ChatEntityService } from './ChatEntityService.js'; import type { OnModuleInit } from '@nestjs/common'; import type { UserEntityService } from './UserEntityService.js'; import type { NoteEntityService } from './NoteEntityService.js'; @@ -56,6 +56,7 @@ export class NotificationEntityService implements OnModuleInit { ) { } + @bindThis onModuleInit() { this.userEntityService = this.moduleRef.get('UserEntityService'); this.noteEntityService = this.moduleRef.get('NoteEntityService'); @@ -68,7 +69,7 @@ export class NotificationEntityService implements OnModuleInit { */ async #packInternal ( src: T, - meId: MiUser['id'], + me: MiUser, options: { checkValidNotifier?: boolean; }, @@ -79,13 +80,13 @@ export class NotificationEntityService implements OnModuleInit { ): Promise | null> { const notification = src; - if (options.checkValidNotifier !== false && !(await this.#isValidNotifier(notification, meId))) return null; + if (options.checkValidNotifier !== false && !(await this.#isValidNotifier(notification, me.id))) return null; const needsNote = NOTE_REQUIRED_NOTIFICATION_TYPES.has(notification.type) && 'noteId' in notification; const noteIfNeed = needsNote ? ( hint?.packedNotes != null ? hint.packedNotes.get(notification.noteId) - : undefOnMissing(this.noteEntityService.pack(notification.noteId, { id: meId }, { + : undefOnMissing(this.noteEntityService.pack(notification.noteId, me, { detail: true, })) ) : undefined; @@ -96,7 +97,7 @@ export class NotificationEntityService implements OnModuleInit { const userIfNeed = needsUser ? ( hint?.packedUsers != null ? hint.packedUsers.get(notification.notifierId) - : undefOnMissing(this.userEntityService.pack(notification.notifierId, { id: meId })) + : undefOnMissing(this.userEntityService.pack(notification.notifierId, me)) ) : undefined; // if the user has been deleted, don't show this notification if (needsUser && !userIfNeed) return null; @@ -106,7 +107,7 @@ export class NotificationEntityService implements OnModuleInit { const reactions = (await Promise.all(notification.reactions.map(async reaction => { const user = hint?.packedUsers != null ? hint.packedUsers.get(reaction.userId)! - : await undefOnMissing(this.userEntityService.pack(reaction.userId, { id: meId })); + : await undefOnMissing(this.userEntityService.pack(reaction.userId, me)); return { user, reaction: reaction.reaction, @@ -131,7 +132,7 @@ export class NotificationEntityService implements OnModuleInit { return packedUser; } - return undefOnMissing(this.userEntityService.pack(userId, { id: meId })); + return undefOnMissing(this.userEntityService.pack(userId, me)); }))).filter(x => x != null); // if all users have been deleted, don't show this notification if (users.length === 0) { @@ -158,7 +159,7 @@ export class NotificationEntityService implements OnModuleInit { } const needsChatRoomInvitation = notification.type === 'chatRoomInvitationReceived'; - const chatRoomInvitation = needsChatRoomInvitation ? await this.chatEntityService.packRoomInvitation(notification.invitationId, { id: meId }).catch(() => null) : undefined; + const chatRoomInvitation = needsChatRoomInvitation ? await this.chatEntityService.packRoomInvitation(notification.invitationId, me).catch(() => null) : undefined; // if the invitation has been deleted, don't show this notification if (needsChatRoomInvitation && !chatRoomInvitation) { return null; @@ -211,36 +212,34 @@ export class NotificationEntityService implements OnModuleInit { async #packManyInternal ( notifications: T[], - meId: MiUser['id'], + me: MiUser, ): Promise { if (notifications.length === 0) return []; let validNotifications = notifications; - validNotifications = await this.#filterValidNotifier(validNotifications, meId); + validNotifications = await this.#filterValidNotifier(validNotifications, me.id); const noteIds = validNotifications.map(x => 'noteId' in x ? x.noteId : null).filter(x => x != null); const notes = noteIds.length > 0 ? await this.notesRepository.find({ where: { id: In(noteIds) }, - relations: ['user', 'reply', 'reply.user', 'renote', 'renote.user'], + relations: ['user', 'reply', 'reply.user', 'renote', 'renote.user', 'renote.reply'], }) : []; - const packedNotesArray = await this.noteEntityService.packMany(notes, { id: meId }, { + const packedNotesArray = await this.noteEntityService.packMany(notes, me, { detail: true, }); const packedNotes = new Map(packedNotesArray.map(p => [p.id, p])); validNotifications = validNotifications.filter(x => !('noteId' in x) || packedNotes.has(x.noteId)); - const userIds = []; + const userIds: string[] = []; for (const notification of validNotifications) { if ('notifierId' in notification) userIds.push(notification.notifierId); if (notification.type === 'reaction:grouped') userIds.push(...notification.reactions.map(x => x.userId)); if (notification.type === 'renote:grouped') userIds.push(...notification.userIds); } - const users = userIds.length > 0 ? await this.usersRepository.find({ - where: { id: In(userIds) }, - }) : []; - const packedUsersArray = await this.userEntityService.packMany(users, { id: meId }); + const users = await this.cacheService.findUsersById(userIds); + const packedUsersArray = await this.userEntityService.packMany(Array.from(users.values()), me); const packedUsers = new Map(packedUsersArray.map(p => [p.id, p])); // 既に解決されたフォローリクエストの通知を除外 @@ -255,7 +254,7 @@ export class NotificationEntityService implements OnModuleInit { const packPromises = validNotifications.map(x => { return this.pack( x, - meId, + me, { checkValidNotifier: false }, { packedNotes, packedUsers }, ); @@ -267,7 +266,7 @@ export class NotificationEntityService implements OnModuleInit { @bindThis public async pack( src: MiNotification | MiGroupedNotification, - meId: MiUser['id'], + me: MiUser, options: { checkValidNotifier?: boolean; @@ -277,23 +276,23 @@ export class NotificationEntityService implements OnModuleInit { packedUsers: Map>; }, ): Promise | null> { - return await this.#packInternal(src, meId, options, hint); + return await this.#packInternal(src, me, options, hint); } @bindThis public async packMany( notifications: MiNotification[], - meId: MiUser['id'], + me: MiUser, ): Promise { - return await this.#packManyInternal(notifications, meId); + return await this.#packManyInternal(notifications, me); } @bindThis public async packGroupedMany( notifications: MiGroupedNotification[], - meId: MiUser['id'], + me: MiUser, ): Promise { - return await this.#packManyInternal(notifications, meId); + return await this.#packManyInternal(notifications, me); } /** @@ -303,12 +302,12 @@ export class NotificationEntityService implements OnModuleInit { notification: T, userIdsWhoMeMuting: Set, userMutedInstances: Set, - notifiers: MiUser[], + notifiers: Map, ): boolean { if (!('notifierId' in notification)) return true; if (userIdsWhoMeMuting.has(notification.notifierId)) return false; - const notifier = notifiers.find(x => x.id === notification.notifierId) ?? null; + const notifier = notifiers.get(notification.notifierId) ?? null; if (notifier == null) return false; if (notifier.host && userMutedInstances.has(notifier.host)) return false; @@ -335,19 +334,18 @@ export class NotificationEntityService implements OnModuleInit { notifications: T[], meId: MiUser['id'], ): Promise { + const notifierIds = notifications.map(notification => 'notifierId' in notification ? notification.notifierId : null).filter(x => x != null); + const [ userIdsWhoMeMuting, userMutedInstances, + notifiers, ] = await Promise.all([ this.cacheService.userMutingsCache.fetch(meId), - this.cacheService.userProfileCache.fetch(meId).then(p => new Set(p.mutedInstances)), + this.cacheService.userMutingsCache.fetch(meId), + this.cacheService.findUsersById(notifierIds), ]); - const notifierIds = notifications.map(notification => 'notifierId' in notification ? notification.notifierId : null).filter(x => x != null); - const notifiers = notifierIds.length > 0 ? await this.usersRepository.find({ - where: { id: In(notifierIds) }, - }) : []; - const filteredNotifications = ((await Promise.all(notifications.map(async (notification) => { const isValid = this.#validateNotifier(notification, userIdsWhoMeMuting, userMutedInstances, notifiers); return isValid ? notification : null; diff --git a/packages/backend/src/core/entities/RoleEntityService.ts b/packages/backend/src/core/entities/RoleEntityService.ts index 3fa38c9521..15b0f06df1 100644 --- a/packages/backend/src/core/entities/RoleEntityService.ts +++ b/packages/backend/src/core/entities/RoleEntityService.ts @@ -14,6 +14,7 @@ import { bindThis } from '@/decorators.js'; import { DEFAULT_POLICIES } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { Packed } from '@/misc/json-schema.js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class RoleEntityService { @@ -25,6 +26,7 @@ export class RoleEntityService { private roleAssignmentsRepository: RoleAssignmentsRepository, private idService: IdService, + private readonly timeService: TimeService, ) { } @@ -40,7 +42,7 @@ export class RoleEntityService { .andWhere(new Brackets(qb => { qb .where('assign.expiresAt IS NULL') - .orWhere('assign.expiresAt > :now', { now: new Date() }); + .orWhere('assign.expiresAt > :now', { now: this.timeService.date }); })) .getCount(); diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index aa85e15258..f080cf1feb 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -44,16 +44,17 @@ import type { UsersRepository, } from '@/models/_.js'; import { bindThis } from '@/decorators.js'; -import { RolePolicies, RoleService } from '@/core/RoleService.js'; -import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; -import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; -import { IdService } from '@/core/IdService.js'; +import { isSystemAccount } from '@/misc/is-system-account.js'; +import type { RolePolicies, RoleService } from '@/core/RoleService.js'; +import type { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; +import type { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; +import type { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { AnnouncementService } from '@/core/AnnouncementService.js'; import type { CustomEmojiService } from '@/core/CustomEmojiService.js'; -import { AvatarDecorationService } from '@/core/AvatarDecorationService.js'; -import { ChatService } from '@/core/ChatService.js'; -import { isSystemAccount } from '@/misc/is-system-account.js'; -import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; +import type { AvatarDecorationService } from '@/core/AvatarDecorationService.js'; +import type { ChatService } from '@/core/ChatService.js'; +import type { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import type { CacheService } from '@/core/CacheService.js'; import { getCallerId } from '@/misc/attach-caller-id.js'; import type { OnModuleInit } from '@nestjs/common'; @@ -153,9 +154,12 @@ export class UserEntityService implements OnModuleInit { @Inject(DI.userMemosRepository) private userMemosRepository: UserMemoRepository, + + private readonly timeService: TimeService, ) { } + @bindThis onModuleInit() { this.apPersonService = this.moduleRef.get('ApPersonService'); this.noteEntityService = this.moduleRef.get('NoteEntityService'); @@ -181,7 +185,9 @@ export class UserEntityService implements OnModuleInit { public validateListenBrainz = ajv.compile(listenbrainzSchema); //#endregion + /** @deprecated use export from MiUser */ public isLocalUser = isLocalUser; + /** @deprecated use export from MiUser */ public isRemoteUser = isRemoteUser; @bindThis @@ -283,7 +289,7 @@ export class UserEntityService implements OnModuleInit { this.cacheService.userBlockingCache.fetch(me), this.cacheService.userMutingsCache.fetch(me), this.cacheService.renoteMutingsCache.fetch(me), - this.cacheService.getUsers(targets) + this.cacheService.findUsersById(targets) .then(users => { const record: Record = {}; for (const [id, user] of users) { @@ -395,7 +401,7 @@ export class UserEntityService implements OnModuleInit { public getOnlineStatus(user: MiUser): 'unknown' | 'online' | 'active' | 'offline' { if (user.hideOnlineStatus) return 'unknown'; if (user.lastActiveDate == null) return 'unknown'; - const elapsed = Date.now() - user.lastActiveDate.getTime(); + const elapsed = this.timeService.now - user.lastActiveDate.getTime(); return ( elapsed < USER_ONLINE_THRESHOLD ? 'online' : elapsed < USER_ACTIVE_THRESHOLD ? 'active' : diff --git a/packages/backend/src/core/entities/UserListEntityService.ts b/packages/backend/src/core/entities/UserListEntityService.ts index b77249c5cb..2722d52195 100644 --- a/packages/backend/src/core/entities/UserListEntityService.ts +++ b/packages/backend/src/core/entities/UserListEntityService.ts @@ -11,6 +11,8 @@ import type { } from '@/models/Blocking.js'; import type { MiUserList } from '@/models/UserList.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; +import { CacheService } from '@/core/CacheService.js'; +import { UserListService } from '@/core/UserListService.js'; import { UserEntityService } from './UserEntityService.js'; @Injectable() @@ -24,25 +26,37 @@ export class UserListEntityService { private userEntityService: UserEntityService, private idService: IdService, + private readonly cacheService: CacheService, + private readonly userListService: UserListService, ) { } @bindThis public async pack( src: MiUserList['id'] | MiUserList, + meId: string | null | undefined, ): Promise> { - const userList = typeof src === 'object' ? src : await this.userListsRepository.findOneByOrFail({ id: src }); + const srcId = typeof(src) === 'object' ? src.id : src; - const users = await this.userListMembershipsRepository.findBy({ - userListId: userList.id, - }); + const [userList, users, favorites] = await Promise.all([ + typeof src === 'object' ? src : this.userListService.userListsCache.fetch(src), + this.cacheService.listUserMembershipsCache.fetch(srcId), + this.cacheService.listUserFavoritesCache.fetch(srcId), + ]); return { id: userList.id, createdAt: this.idService.parse(userList.id).date.toISOString(), + createdBy: userList.userId, name: userList.name, - userIds: users.map(x => x.userId), + userIds: users.keys().toArray(), isPublic: userList.isPublic, + isLiked: meId != null + ? favorites.has(meId) + : undefined, + likedCount: userList.isPublic || meId === userList.userId + ? favorites.size + : undefined, }; } diff --git a/packages/backend/src/daemons/ApLogCleanupService.ts b/packages/backend/src/daemons/ApLogCleanupService.ts index 61f76b4e2c..d3f09bf660 100644 --- a/packages/backend/src/daemons/ApLogCleanupService.ts +++ b/packages/backend/src/daemons/ApLogCleanupService.ts @@ -8,6 +8,7 @@ import { bindThis } from '@/decorators.js'; import { LoggerService } from '@/core/LoggerService.js'; import Logger from '@/logger.js'; import { ApLogService } from '@/core/ApLogService.js'; +import { TimeService, type TimerHandle } from '@/global/TimeService.js'; // 10 minutes export const scanInterval = 1000 * 60 * 10; @@ -15,10 +16,12 @@ export const scanInterval = 1000 * 60 * 10; @Injectable() export class ApLogCleanupService implements OnApplicationShutdown { private readonly logger: Logger; - private scanTimer: NodeJS.Timeout | null = null; + private scanTimer: TimerHandle | null = null; constructor( private readonly apLogService: ApLogService, + private readonly timeService: TimeService, + loggerService: LoggerService, ) { this.logger = loggerService.getLogger('activity-log-cleanup'); @@ -34,7 +37,7 @@ export class ApLogCleanupService implements OnApplicationShutdown { this.tick(); // Prune on a regular interval for the lifetime of the server. - this.scanTimer = setInterval(this.tick, scanInterval); + this.scanTimer = this.timeService.startTimer(this.tick, scanInterval, { repeated: true }); } @bindThis @@ -55,7 +58,7 @@ export class ApLogCleanupService implements OnApplicationShutdown { @bindThis public dispose(): void { if (this.scanTimer) { - clearInterval(this.scanTimer); + this.timeService.stopTimer(this.scanTimer); this.scanTimer = null; } } diff --git a/packages/backend/src/daemons/DaemonModule.ts b/packages/backend/src/daemons/DaemonModule.ts index ea71875f19..286fba56f3 100644 --- a/packages/backend/src/daemons/DaemonModule.ts +++ b/packages/backend/src/daemons/DaemonModule.ts @@ -5,14 +5,12 @@ import { Module } from '@nestjs/common'; import { CoreModule } from '@/core/CoreModule.js'; -import { GlobalModule } from '@/GlobalModule.js'; import { QueueStatsService } from './QueueStatsService.js'; import { ServerStatsService } from './ServerStatsService.js'; import { ApLogCleanupService } from './ApLogCleanupService.js'; @Module({ imports: [ - GlobalModule, CoreModule, ], providers: [ diff --git a/packages/backend/src/daemons/QueueStatsService.ts b/packages/backend/src/daemons/QueueStatsService.ts index 9888ae3942..3779172517 100644 --- a/packages/backend/src/daemons/QueueStatsService.ts +++ b/packages/backend/src/daemons/QueueStatsService.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import Xev from 'xev'; import * as Bull from 'bullmq'; import { QueueService } from '@/core/QueueService.js'; +import { TimeService, type TimerHandle } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; @@ -31,7 +32,7 @@ const interval = 10000; @Injectable() export class QueueStatsService implements OnApplicationShutdown { - private intervalId?: NodeJS.Timeout; + private intervalId?: TimerHandle; private activeDeliverJobs = 0; private activeInboxJobs = 0; @@ -45,6 +46,7 @@ export class QueueStatsService implements OnApplicationShutdown { private config: Config, private queueService: QueueService, + private readonly timeService: TimeService, ) { } @@ -114,13 +116,13 @@ export class QueueStatsService implements OnApplicationShutdown { tick(); - this.intervalId = setInterval(tick, interval); + this.intervalId = this.timeService.startTimer(tick, interval, { repeated: true }); } @bindThis public async stop() { if (this.intervalId) { - clearInterval(this.intervalId); + this.timeService.stopTimer(this.intervalId); } this.log = undefined; diff --git a/packages/backend/src/daemons/ServerStatsService.ts b/packages/backend/src/daemons/ServerStatsService.ts index 9fafa54b04..8c93ad9818 100644 --- a/packages/backend/src/daemons/ServerStatsService.ts +++ b/packages/backend/src/daemons/ServerStatsService.ts @@ -11,6 +11,7 @@ import { bindThis } from '@/decorators.js'; import type { OnApplicationShutdown } from '@nestjs/common'; import { MiMeta } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; +import { TimeService, type TimerHandle } from '@/global/TimeService.js'; export interface Stats { cpu: number, @@ -37,13 +38,14 @@ const round = (num: number) => Math.round(num * 10) / 10; @Injectable() export class ServerStatsService implements OnApplicationShutdown { - private intervalId: NodeJS.Timeout | null = null; + private intervalId: TimerHandle | null = null; private log: Stats[] = []; constructor( @Inject(DI.meta) private meta: MiMeta, + private readonly timeService: TimeService, ) { } @@ -90,13 +92,13 @@ export class ServerStatsService implements OnApplicationShutdown { tick(); - this.intervalId = setInterval(tick, interval); + this.intervalId = this.timeService.startTimer(tick, interval, { repeated: true }); } @bindThis public dispose(): void { if (this.intervalId) { - clearInterval(this.intervalId); + this.timeService.stopTimer(this.intervalId); } this.log = []; diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index 099d48c81a..e2c73562c8 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -14,6 +14,7 @@ export const DI = { redisForTimelines: Symbol('redisForTimelines'), redisForReactions: Symbol('redisForReactions'), redisForRateLimit: Symbol('redisForRateLimit'), + console: Symbol('console'), //#region Repositories usersRepository: Symbol('usersRepository'), diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 9a50eb8561..49a734113a 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -3,7 +3,19 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -const envOption = { +export interface EnvOption { + onlyQueue: boolean; + onlyServer: boolean; + noDaemons: boolean; + disableClustering: boolean; + verbose: boolean; + withLogTime: boolean; + quiet: boolean; + hideWorkerId: boolean; + [key: string]: boolean; +} + +const defaultEnvOption: Readonly = { onlyQueue: false, onlyServer: false, noDaemons: false, @@ -14,12 +26,67 @@ const envOption = { hideWorkerId: false, }; -for (const key of Object.keys(envOption) as (keyof typeof envOption)[]) { - if (process.env['MK_' + key.replace(/[A-Z]/g, letter => `_${letter}`).toUpperCase()]) envOption[key] = true; +function translateKey(key: string): string { + return 'MK_' + key.replace(/[A-Z]/g, letter => `_${letter}`).toUpperCase(); } -if (process.env.NODE_ENV === 'test') envOption.disableClustering = true; -if (process.env.NODE_ENV === 'test') envOption.quiet = true; -if (process.env.NODE_ENV === 'test') envOption.noDaemons = true; +const testEnvOption: Readonly = { + ...defaultEnvOption, + disableClustering: true, + quiet: true, + noDaemons: true, +}; -export { envOption }; +/** @deprecated use EnvService when possible */ +export const envOption: EnvOption = createEnvOptions(() => process.env); + +export function createEnvOptions(getEnv: () => Partial>): EnvOption { + return new Proxy({} as EnvOption, { + get(target, key) { + if (typeof(key) !== 'string') { + return Reflect.get(target, key); + } + + const env = getEnv(); + const envKey = translateKey(key); + if (envKey in env) { + const envValue = env[envKey]?.toLowerCase(); + return !!envValue && envValue !== '0' && envValue !== 'false'; + } + + const def = env.NODE_ENV === 'test' ? testEnvOption : defaultEnvOption; + if (key in def) { + return def[key]; + } + + return false; + }, + set(target, key, value) { + if (typeof(key) !== 'string') { + return Reflect.set(target, key, value); + } + + const env = getEnv(); + const envKey = translateKey(key); + if (value) { + env[envKey] = '1'; + } else { + delete env[envKey]; + } + return true; + }, + has(target, key): boolean { + return typeof(key) === 'string' || key in target; + }, + deleteProperty(target, key): boolean { + if (typeof(key) !== 'string') { + return Reflect.deleteProperty(target, key); + } + + const env = getEnv(); + const envKey = translateKey(key); + delete env[envKey]; + return true; + }, + }); +} diff --git a/packages/backend/src/global/CacheManagementService.ts b/packages/backend/src/global/CacheManagementService.ts new file mode 100644 index 0000000000..f350d765b7 --- /dev/null +++ b/packages/backend/src/global/CacheManagementService.ts @@ -0,0 +1,167 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable, type OnApplicationShutdown } from '@nestjs/common'; +import { + MemoryKVCache, + MemorySingleCache, + RedisKVCache, + RedisSingleCache, + type RedisKVCacheOpts, + type RedisSingleCacheOpts, + type MemoryCacheServices, + type RedisCacheServices, + type MemoryCacheOpts, +} from '@/misc/cache.js'; +import { + QuantumKVCache, + type QuantumKVOpts, + type QuantumCacheServices, +} from '@/misc/QuantumKVCache.js'; +import { bindThis } from '@/decorators.js'; +import { DI } from '@/di-symbols.js'; +import { TimeService, type TimerHandle } from '@/global/TimeService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { callAllOn, callAllOnAsync } from '@/misc/call-all.js'; +import type * as Redis from 'ioredis'; + +// This is the one place that's *supposed* to new() up caches. +/* eslint-disable no-restricted-syntax */ + +export type ManagedMemoryKVCache = Managed>; +export type ManagedMemorySingleCache = Managed>; +export type ManagedRedisKVCache = Managed>; +export type ManagedRedisSingleCache = Managed>; +export type ManagedQuantumKVCache = Managed>; + +export type Managed = Omit; +export type Manager = { dispose(): Promise | void, clear(): void, gc(): void }; + +type CacheServices = MemoryCacheServices & RedisCacheServices & QuantumCacheServices; + +export const GC_INTERVAL = 1000 * 60 * 3; // 3m + +/** + * Creates and "manages" instances of any standard cache type. + * Instances produced by this class are automatically tracked for disposal when the application shuts down. + */ +@Injectable() +export class CacheManagementService implements OnApplicationShutdown { + private readonly managedCaches = new Map(); + private gcTimer?: TimerHandle | null; + + constructor( + @Inject(DI.redis) + private readonly redisClient: Redis.Redis, + + private readonly timeService: TimeService, + private readonly internalEventService: InternalEventService, + ) {} + + private get cacheServices(): CacheServices { + return { + internalEventService: this.internalEventService, + redisClient: this.redisClient, + timeService: this.timeService, + }; + } + + @bindThis + public createMemoryKVCache(name: string, optsOrLifetime: MemoryCacheOpts | number): ManagedMemoryKVCache { + const opts = typeof(optsOrLifetime) === 'number' ? { lifetime: optsOrLifetime } : optsOrLifetime; + return this.create(name, () => new MemoryKVCache(name, this.cacheServices, opts)); + } + + @bindThis + public createMemorySingleCache(name: string, optsOrLifetime: MemoryCacheOpts | number): ManagedMemorySingleCache { + const opts = typeof(optsOrLifetime) === 'number' ? { lifetime: optsOrLifetime } : optsOrLifetime; + return this.create(name, () => new MemorySingleCache(name, this.cacheServices, opts)); + } + + @bindThis + public createRedisKVCache(name: string, opts: RedisKVCacheOpts): ManagedRedisKVCache { + return this.create(name, () => new RedisKVCache(name, this.cacheServices, opts)); + } + + @bindThis + public createRedisSingleCache(name: string, opts: RedisSingleCacheOpts): ManagedRedisSingleCache { + return this.create(name, () => new RedisSingleCache(name, this.cacheServices, opts)); + } + + @bindThis + public createQuantumKVCache(name: string, opts: QuantumKVOpts): ManagedQuantumKVCache { + return this.create(name, () => new QuantumKVCache(name, this.cacheServices, opts)); + } + + private create(name: string, factory: () => T): T { + if (this.managedCaches.has(name)) { + throw new Error(`Duplicate cache name: "${name}"`); + } + + const cache = factory(); + + this.managedCaches.set(name, cache); + this.startGcTimer(); + + return cache; + } + + @bindThis + public gc(): void { + this.resetGcTimer(() => { + callAllOn(this.managedCaches.values(), 'gc'); + }); + } + + @bindThis + public clear(): void { + this.resetGcTimer(() => { + callAllOn(this.managedCaches.values(), 'clear'); + }); + } + + @bindThis + public async dispose(): Promise { + this.stopGcTimer(); + + const toDispose = Array.from(this.managedCaches.values()); + this.managedCaches.clear(); + + await callAllOnAsync(toDispose, 'dispose'); + } + + @bindThis + public async onApplicationShutdown(): Promise { + await this.dispose(); + } + + @bindThis + private startGcTimer() { + // Only start it once, and don't *re* start since this gets called repeatedly. + this.gcTimer ??= this.timeService.startTimer(this.gc, GC_INTERVAL, { repeated: true }); + } + + @bindThis + private stopGcTimer() { + // Only stop it once, then clear the value so it can be restarted later. + if (this.gcTimer != null) { + this.timeService.stopTimer(this.gcTimer); + this.gcTimer = null; + } + } + + @bindThis + private resetGcTimer(onBlank?: () => void): void { + this.stopGcTimer(); + + try { + if (onBlank) { + onBlank(); + } + } finally { + this.startGcTimer(); + } + } +} diff --git a/packages/backend/src/global/DependencyService.ts b/packages/backend/src/global/DependencyService.ts new file mode 100644 index 0000000000..929fc319e2 --- /dev/null +++ b/packages/backend/src/global/DependencyService.ts @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import nodePath from 'node:path'; +import nodeFs from 'node:fs'; +import { Injectable } from '@nestjs/common'; +import { CacheManagementService, type ManagedMemoryKVCache } from '@/global/CacheManagementService.js'; +import { bindThis } from '@/decorators.js'; + +@Injectable() +export class DependencyService { + protected readonly dependencyVersionCache: ManagedMemoryKVCache; + + constructor(cacheManagementService: CacheManagementService) { + this.dependencyVersionCache = cacheManagementService.createMemoryKVCache('dependencyVersion', Infinity); + } + + /** + * Returns the installed version of a given dependency, or null if not installed. + */ + @bindThis + public async getDependencyVersion(dependency: string): Promise { + return await this.dependencyVersionCache.fetch(dependency, async () => { + const packageJsonPath = nodePath.join(import.meta.dirname, '../../package.json'); + const packageJsonText = nodeFs.readFileSync(packageJsonPath, 'utf8'); + + // No "dependencies" section -> infer not installed. + const packageJson = JSON.parse(packageJsonText) as { dependencies?: Partial> }; + if (packageJson.dependencies == null) return null; + + // Not listed -> not installed. + const version = packageJson.dependencies['mfm-js']; + if (version == null) return null; + + // Just in case some other value is there + return String(version); + }); + } +} diff --git a/packages/backend/src/global/EnvService.ts b/packages/backend/src/global/EnvService.ts new file mode 100644 index 0000000000..bb73ff57f9 --- /dev/null +++ b/packages/backend/src/global/EnvService.ts @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import { createEnvOptions, type EnvOption } from '@/env.js'; + +/** + * Provides structured, mockable access to runtime/environment details. + */ +@Injectable() +export class EnvService { + protected readonly envOptions: EnvOption = createEnvOptions(() => this.env); + + /** + * Returns the environment variables of the process. + * Modifications are reflected back to the local process, but not to the operating system environment. + */ + public get env(): Partial> { + return process.env; + } + + /** + * Maps and returns environment-based options for the process. + * Modifications are reflected back to the local process ("env" property), but not to the operating system environment. + */ + public get options(): EnvOption { + return this.envOptions; + } +} diff --git a/packages/backend/src/core/InternalEventService.ts b/packages/backend/src/global/InternalEventService.ts similarity index 61% rename from packages/backend/src/core/InternalEventService.ts rename to packages/backend/src/global/InternalEventService.ts index 5b164b605e..f2af08b036 100644 --- a/packages/backend/src/core/InternalEventService.ts +++ b/packages/backend/src/global/InternalEventService.ts @@ -4,13 +4,14 @@ */ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; -import Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; -import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; +import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js'; +import type { Config } from '@/config.js'; +import type Redis from 'ioredis'; -export type Listener = (value: InternalEventTypes[K], key: K, isLocal: boolean) => void | Promise; +export type EventTypes = InternalEventTypes; +export type Listener = (value: EventTypes[K], key: K, isLocal: boolean) => void | Promise; export interface ListenerProps { ignoreLocal?: boolean, @@ -19,19 +20,23 @@ export interface ListenerProps { @Injectable() export class InternalEventService implements OnApplicationShutdown { - private readonly listeners = new Map, ListenerProps>>(); + private readonly listeners = new Map, ListenerProps>>(); constructor( @Inject(DI.redisForSub) private readonly redisForSub: Redis.Redis, - private readonly globalEventService: GlobalEventService, + @Inject(DI.redis) + private readonly redisForPub: Redis.Redis, + + @Inject(DI.config) + private readonly config: Pick, ) { this.redisForSub.on('message', this.onMessage); } @bindThis - public on(type: K, listener: Listener, props?: ListenerProps): void { + public on(type: K, listener: Listener, props?: ListenerProps): void { let set = this.listeners.get(type); if (!set) { set = new Map(); @@ -39,22 +44,25 @@ export class InternalEventService implements OnApplicationShutdown { } // Functionally, this is just a set with metadata on the values. - set.set(listener as Listener, props ?? {}); + set.set(listener as Listener, props ?? {}); } @bindThis - public off(type: K, listener: Listener): void { - this.listeners.get(type)?.delete(listener as Listener); + public off(type: K, listener: Listener): void { + this.listeners.get(type)?.delete(listener as Listener); } @bindThis - public async emit(type: K, value: InternalEventTypes[K]): Promise { + public async emit(type: K, value: EventTypes[K]): Promise { await this.emitInternal(type, value, true); - await this.globalEventService.publishInternalEventAsync(type, { ...value, _pid: process.pid }); + await this.redisForPub.publish(this.config.host, JSON.stringify({ + channel: 'internal', + message: { type: type, body: value }, + })); } @bindThis - private async emitInternal(type: K, value: InternalEventTypes[K], isLocal: boolean): Promise { + private async emitInternal(type: K, value: EventTypes[K], isLocal: boolean): Promise { const listeners = this.listeners.get(type); if (!listeners) { return; @@ -77,7 +85,7 @@ export class InternalEventService implements OnApplicationShutdown { if (obj.channel === 'internal') { const { type, body } = obj.message as GlobalEvents['internal']['payload']; if (!isLocalInternalEvent(body) || body._pid !== process.pid) { - await this.emitInternal(type, body as InternalEventTypes[keyof InternalEventTypes], false); + await this.emitInternal(type, body as EventTypes[keyof EventTypes], false); } } } diff --git a/packages/backend/src/global/TimeService.ts b/packages/backend/src/global/TimeService.ts new file mode 100644 index 0000000000..d1f6d98358 --- /dev/null +++ b/packages/backend/src/global/TimeService.ts @@ -0,0 +1,186 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable, type OnApplicationShutdown } from '@nestjs/common'; +import { bindThis } from '@/decorators.js'; +import { withSignal, withCleanup } from '@/misc/promiseUtils.js'; + +const timerTokenSymbol = Symbol('timerToken'); + +/** + * Provides abstractions to access the current time. + * Exists for unit testing purposes, so that tests can "simulate" any given time for consistency. + */ +@Injectable() +export abstract class TimeService implements OnApplicationShutdown { + protected readonly timers = new Map(); + + protected constructor() {} + + /** + * Returns the current time, in milliseconds since the Unix epoch. + */ + public abstract get now(): number; + + /** + * Returns a new Date instance representing the current time. + */ + public get date(): Date { + return new Date(this.now); + } + + public startTimer(callback: () => void, delay: number, opts?: TimerOpts): TimerHandle; + public startTimer(callback: (value: T) => void, delay: number, opts: TimerOpts | undefined, value: T): TimerHandle; + @bindThis + public startTimer(callback: (value: T) => void, delay: number, opts?: TimerOpts, value?: T): TimerHandle { + const timerId = Symbol(); + const repeating = opts?.repeated ?? false; + + const timer = this.startNativeTimer(timerId, repeating, () => { + callback(value as T); // overloads ensure it can't be null + }, delay); + this.timers.set(timerId, timer); + + return timerId; + } + + public startPromiseTimer(delay: number): PromiseTimerHandle; + public startPromiseTimer(delay: number, value: T, opts?: PromiseTimerOpts): PromiseTimerHandle; + @bindThis + public startPromiseTimer(delay: number, value?: T, opts?: PromiseTimerOpts): PromiseTimerHandle { + const timerId = Symbol(); + const abortController = new AbortController(); + const abortSignal = opts?.signal ? AbortSignal.any([abortController.signal, opts.signal]) : abortController.signal; + + const handlePromise = + withCleanup( + // Bind abort signal + withSignal( + () => new Promise(resolve => { + // Start the underlying timer + this.startTimer(resolve, delay, undefined, value as T); // overloads ensure it can't be null + }), + abortSignal, + ), + + // Register cleanup func + () => { + // Make sure we dispose the real handle if promise rejects! + this.stopTimer(timerId); + }); + + // Populate and return the handle. + return Object.assign(handlePromise, { + [timerTokenSymbol]: timerId, + + abort: (reason: Error) => { + abortController.abort(reason); + }, + }); + } + + protected abstract startNativeTimer(timerId: symbol, repeating: boolean, callback: () => void, delay: number): TTimer; + + /** + * Clears a registered timeout or interval. + * Returns true if the registration exists and was still active, false otherwise. + * Safe to call with invalid or expired IDs. + */ + @bindThis + public stopTimer(handle: TimerHandle | PromiseTimerHandle): boolean { + const id = typeof(handle) === 'object' ? handle[timerTokenSymbol] : handle; + const reg = this.timers.get(id); + if (!reg) return false; + + this.stopNativeTimer(reg); + this.timers.delete(id); + return true; + } + + protected abstract stopNativeTimer(reg: TTimer): void; + + /** + * Cleanup all handles and references. + * Safe to call multiple times. + * + * **Must be called before shutting down the app!** + */ + @bindThis + public dispose(): void { + for (const reg of this.timers.values()) { + this.stopNativeTimer(reg); + } + this.timers.clear(); + } + + @bindThis + onApplicationShutdown(): void { + this.dispose(); + } +} + +export interface Timer { + timerId: symbol; + repeating: boolean; + delay: number; + callback: () => void; +} + +export interface TimerOpts { + repeated?: boolean; +} + +export type TimerHandle = symbol; + +export interface PromiseTimerOpts { + signal?: AbortSignal; +} + +export interface PromiseTimerHandle extends PromiseLike { + readonly [timerTokenSymbol]: symbol; + abort(error?: Error): void; +} + +/** + * Default implementation of TimeService, uses Date.now() as time source and setTimeout/setInterval for timers. + */ +@Injectable() +export class NativeTimeService extends TimeService implements OnApplicationShutdown { + public get now(): number { + // This is the one place that actually *should* have it + // eslint-disable-next-line no-restricted-properties + return Date.now(); + } + + public constructor() { + super(); + } + + protected startNativeTimer(timerId: symbol, repeating: boolean, callback: () => void, delay: number): NativeTimer { + // Wrap the caller's callback to make sure we clean up the registration. + const wrappedCallback = () => { + this.timers.delete(timerId); + callback(); + }; + + const timeout = repeating + ? global.setInterval(wrappedCallback, delay) + : global.setTimeout(wrappedCallback, delay); + + return { callback, timerId, repeating, delay, timeout }; + } + + protected stopNativeTimer(reg: NativeTimer): void { + if (reg.repeating) { + global.clearInterval(reg.timeout); + } else { + global.clearTimeout(reg.timeout); + } + } +} + +export interface NativeTimer extends Timer { + timeout: NodeJS.Timeout; +} diff --git a/packages/backend/src/logger.ts b/packages/backend/src/logger.ts index 4bf45fc76b..6de0b7ffc0 100644 --- a/packages/backend/src/logger.ts +++ b/packages/backend/src/logger.ts @@ -8,7 +8,8 @@ import chalk from 'chalk'; import { default as convertColor } from 'color-convert'; import { format as dateFormat } from 'date-fns'; import { bindThis } from '@/decorators.js'; -import { envOption } from './env.js'; +import { TimeService, NativeTimeService } from '@/global/TimeService.js'; +import { EnvService } from '@/global/EnvService.js'; import type { KEYWORD } from 'color-convert/conversions.js'; type Context = { @@ -23,45 +24,65 @@ export type DataElement = DataObject | Error | string | null; // https://stackoverflow.com/questions/61148466/typescript-type-that-matches-any-object-but-not-arrays export type DataObject = Record | (object & { length?: never; }); +export type Console = Pick; +export const nativeConsole: Console = global.console; + +const fallbackTimeService = new NativeTimeService(); +const fallbackEnvService = new EnvService(); + const levelFuncs = { error: 'error', warning: 'warn', success: 'info', info: 'log', debug: 'debug', -} as const satisfies Record; +} as const satisfies Record; // eslint-disable-next-line import/no-default-export export default class Logger { private context: Context; private parentLogger: Logger | null = null; - public readonly verbose: boolean; + private readonly timeService: TimeService; + private readonly envService: EnvService; - constructor(context: string, color?: KEYWORD, verbose?: boolean) { + /** + * Where to send the actual log strings. + * Defaults to the native global.console instance. + */ + private readonly console: Console; + + constructor(context: string, color?: KEYWORD, envService?: EnvService, timeService?: TimeService, console?: Console) { this.context = { name: context, color: color, }; - this.verbose = verbose ?? envOption.verbose; + this.envService = envService ?? fallbackEnvService; + this.console = console ?? nativeConsole; + this.timeService = timeService ?? fallbackTimeService; } @bindThis public createSubLogger(context: string, color?: KEYWORD): Logger { - const logger = new Logger(context, color, this.verbose); + const logger = new Logger(context, color, this.envService, this.timeService, this.console); logger.parentLogger = this; return logger; } @bindThis private log(level: Level, message: string, data?: Data, important = false, subContexts: Context[] = []): void { - if (envOption.quiet) return; + if (this.envService.options.quiet) return; + + // Debugging logging is disabled in production unless MK_VERBOSE is set. + if (level === 'debug' && this.envService.env.NODE_ENV === 'production' && !this.envService.options.verbose) { + return; + } if (this.parentLogger) { this.parentLogger.log(level, message, data, important, [this.context].concat(subContexts)); return; } - const time = dateFormat(new Date(), 'HH:mm:ss'); + const time = dateFormat(this.timeService.date, 'HH:mm:ss'); const worker = cluster.isPrimary ? '*' : cluster.worker!.id; const l = level === 'error' ? important ? chalk.bgRed.white('ERR ') : chalk.red('ERR ') : @@ -79,10 +100,10 @@ export default class Logger { level === 'info' ? message : null; - let log = envOption.hideWorkerId + let log = this.envService.options.hideWorkerId ? `${l}\t[${contexts.join(' ')}]\t\t${m}` : `${l} ${worker}\t[${contexts.join(' ')}]\t\t${m}`; - if (envOption.withLogTime) log = chalk.gray(time) + ' ' + log; + if (this.envService.options.withLogTime) log = chalk.gray(time) + ' ' + log; const args: unknown[] = [important ? chalk.bold(log) : log]; if (Array.isArray(data)) { @@ -94,7 +115,7 @@ export default class Logger { } else if (data != null) { args.push(data); } - console[levelFuncs[level]](...args); + this.console[levelFuncs[level]](...args); } @bindThis @@ -122,9 +143,7 @@ export default class Logger { @bindThis public debug(message: string, data?: Data, important = false): void { // デバッグ用に使う(開発者に必要だが利用者に不要な情報) - if (process.env.NODE_ENV !== 'production' || this.verbose) { - this.log('debug', message, data, important); - } + this.log('debug', message, data, important); } @bindThis diff --git a/packages/backend/src/misc/QuantumKVCache.ts b/packages/backend/src/misc/QuantumKVCache.ts index b96937d6f2..6f0ca25450 100644 --- a/packages/backend/src/misc/QuantumKVCache.ts +++ b/packages/backend/src/misc/QuantumKVCache.ts @@ -3,36 +3,159 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { InternalEventService } from '@/core/InternalEventService.js'; +import { EntityNotFoundError } from 'typeorm'; +import promiseLimit from 'promise-limit'; import { bindThis } from '@/decorators.js'; -import { InternalEventTypes } from '@/core/GlobalEventService.js'; -import { MemoryKVCache } from '@/misc/cache.js'; +import type { InternalEventService, EventTypes } from '@/global/InternalEventService.js'; +import { MemoryKVCache, type MemoryCacheServices } from '@/misc/cache.js'; +import { makeKVPArray, type KVPArray } from '@/misc/kvp-array.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; +import { FetchFailedError } from '@/misc/errors/FetchFailedError.js'; +import { KeyNotFoundError } from '@/misc/errors/KeyNotFoundError.js'; +import { QuantumCacheError } from '@/misc/errors/QuantumCacheError.js'; +import { DisposedError, DisposingError } from '@/misc/errors/DisposeError.js'; +import { trackPromise } from '@/misc/promise-tracker.js'; +import { withCleanup, withSignal } from '@/misc/promiseUtils.js'; +import { promiseTry } from '@/misc/promise-try.js'; -export interface QuantumKVOpts { +export interface QuantumKVOpts = Value> { /** * Memory cache lifetime in milliseconds. */ lifetime: number; /** - * Callback to fetch the value for a key that wasn't found in the cache. - * May be synchronous or async. + * Callback to fetch required values by key. */ - fetcher: (key: string, cache: QuantumKVCache) => T | Promise; + fetcher: Fetcher; /** - * Optional callback to fetch the value for multiple keys that weren't found in the cache. - * May be synchronous or async. - * If not provided, then the implementation will fall back on repeated calls to fetcher(). + * Callback to fetch optional values by key. */ - bulkFetcher?: (keys: string[], cache: QuantumKVCache) => Iterable<[key: string, value: T]> | Promise>; + optionalFetcher?: OptionalFetcher; /** - * Optional callback when one or more values are changed (created, updated, or deleted) in the cache, either locally or elsewhere in the cluster. - * This is called *after* the cache state is updated. - * Implementations may be synchronous or async. + * Callback to fetch multiple optional values by key. */ - onChanged?: (keys: string[], cache: QuantumKVCache) => void | Promise; + bulkFetcher?: BulkFetcher; + + /** + * Callback to handle changes to the cross-cluster state (create, update, or delete values). + */ + onChanged?: OnChanged; + + /** + * Callback to handle a whole-state reset (all values deleted). + */ + onReset?: OnReset; + + /** + * Optional limit on the number of calls to fetcher to allow at once. + * If more than this many fetches are attempted, the excess will be queued until the earlier operations complete. + * The total number of calls will never exceed maxConcurrency. + * Min: 1 + * Default: 4 + */ + fetcherConcurrency?: number; + + /** + * Optional limit on the number of calls to optionalFetcher to allow at once. + * If more than this many fetches are attempted, the excess will be queued until the earlier operations complete. + * The total number of calls will never exceed maxConcurrency. + * Min: 1 + * Default: 4 + */ + optionalFetcherConcurrency?: number; + + /** + * Optional limit on the number of calls to bulkFetcher to allow at once. + * If more than this many fetches are attempted, the excess will be queued until the earlier operations complete. + * The total number of calls will never exceed maxConcurrency. + * Min: 1 + * Default: 2 + */ + bulkFetcherConcurrency?: number; + + /** + * Optional limit on the total number of calls to fetcher, optionalFetcher, or bulkFetcher to allow at once. + * If more than this many fetches are attempted, the excess will be queued until the earlier operations complete. + * Min: 1 + * Default: fetcherConcurrency, optionalFetcherConcurrency, or bulkFetcherConcurrency - whichever is highest. + */ + maxConcurrency?: number; +} + +export interface CallbackMeta { + /** + * The cache instance that triggered this callback. + */ + readonly cache: QuantumKVCache; + + /** + * AbortSignal that will fire when the cache is disposed. + * Should be propagated to ensure smooth cleanup and shutdown. + */ + readonly disposeSignal: AbortSignal; +} + +/** + * Callback to fetch the value for a key that wasn't found in the cache, and is required to continue. + * Should return the fetched value, or null/undefined if no value exists for the given key. + * Missing keys may also produce an EntityNotFound or KeyNotFoundException exception, which will be wrapped to gracefully abort the operation. + * May be synchronous or async. + */ +export type Fetcher = (key: string, meta: CallbackMeta) => MaybePromise | null | undefined>; + +/** + * Optional callback to fetch the value for a key that wasn't found in the cache, and isn't required to continue. + * Should return the fetched value, or null/undefined if no value exists for the given key. + * Missing keys should *not* produce any exception, as it will be wrapped to gracefully abort the operation. + * May be synchronous or async. + * If not provided, then the implementation will fall back on fetcher(). + */ +export type OptionalFetcher = (key: string, meta: CallbackMeta) => MaybePromise | null | undefined>; + +/** + * Optional callback to fetch the value for multiple keys that weren't found in the cache. + * Should return the fetched values for each key, or null/undefined if no value exists for the given key. + * Missing keys may also be omitted from the response entirely, but no error should be thrown. + * May be synchronous or async. + * If not provided, then the implementation will fall back on repeated calls to optionalFetcher() or fetcher(). + */ +export type BulkFetcher = (keys: string[], meta: CallbackMeta) => MaybePromise | null | undefined]>>; + +/** + * Optional callback when one or more values are changed (created, updated, or deleted) in the cache, either locally or elsewhere in the cluster. + * This is called *after* the cache state is updated. + * May be synchronous or async. + */ +export type OnChanged = (keys: string[], meta: CallbackMeta) => MaybePromise; + +/** + * Optional callback when all values are removed from the cache, either locally or elsewhere in the cluster. + * This is called *after* the cache state is updated. + * May be synchronous or async. + */ +export type OnReset = (meta: CallbackMeta) => MaybePromise; + +type ActiveFetcher = Promise; +type ActiveOptionalFetcher = Promise; +type ActiveBulkFetcher = Promise[]>; + +// Make sure null / undefined cannot be a valid type +// https://stackoverflow.com/a/63045455 +type Value = NonNullable; +type KeyValue = [key: string, value: T]; +type Limiter = (callback: () => Promise) => Promise; +type MaybePromise = T | Promise; +type AtLeastOne = [T, ...T[]]; + +export interface QuantumCacheServices extends MemoryCacheServices { + /** + * Event bus to attach to. + * This can be mocked for easier testing under DI. + */ + readonly internalEventService: InternalEventService; } /** @@ -40,32 +163,91 @@ export interface QuantumKVOpts { * All nodes in the cluster are guaranteed to have a *subset* view of the current accurate state, though individual processes may have different items in their local cache. * This ensures that a call to get() will never return stale data. */ -export class QuantumKVCache implements Iterable<[key: string, value: T]> { +export class QuantumKVCache = Value> implements Iterable { + private readonly internalEventService: InternalEventService; + private readonly memoryCache: MemoryKVCache; - public readonly fetcher: QuantumKVOpts['fetcher']; - public readonly bulkFetcher: QuantumKVOpts['bulkFetcher']; - public readonly onChanged: QuantumKVOpts['onChanged']; + private readonly activeFetchers = new Map>(); + private readonly activeOptionalFetchers = new Map>(); + private readonly activeBulkFetchers = new Map>(); + + private readonly globalLimiter: Limiter; + private readonly fetcherLimiter: Limiter; + private readonly optionalFetcherLimiter: Limiter; + private readonly bulkFetcherLimiter: Limiter; + + public readonly fetcher: Fetcher; + public readonly optionalFetcher: OptionalFetcher | undefined; + public readonly bulkFetcher: BulkFetcher | undefined; + public readonly onChanged: OnChanged | undefined; + public readonly onReset: OnReset | undefined; + + private readonly disposeController = new AbortController(); + private isDisposing = false; + private isDisposed = false; /** - * @param internalEventService Service bus to synchronize events. * @param name Unique name of the cache - must be the same in all processes. + * @param services DI services - internalEventService is required * @param opts Cache options */ constructor( - private readonly internalEventService: InternalEventService, - private readonly name: string, - opts: QuantumKVOpts, + public readonly name: string, + services: QuantumCacheServices, + opts: QuantumKVOpts, ) { - this.memoryCache = new MemoryKVCache(opts.lifetime); + // OK: we forward all management calls to the inner cache. + // eslint-disable-next-line no-restricted-syntax + this.memoryCache = new MemoryKVCache(name + ':mem', services, { lifetime: opts.lifetime }); + + // Set up rate limiters + const fetcherConcurrency = opts.fetcherConcurrency + ? Math.max(opts.fetcherConcurrency, 1) + : 4; + this.fetcherLimiter = promiseLimit(fetcherConcurrency); + + const optionalFetcherConcurrency = opts.optionalFetcherConcurrency + ? Math.max(opts.optionalFetcherConcurrency, 1) + : 4; + this.optionalFetcherLimiter = promiseLimit(optionalFetcherConcurrency); + + const bulkFetcherConcurrency = opts.bulkFetcherConcurrency + ? Math.max(opts.bulkFetcherConcurrency, 1) + : 2; + this.bulkFetcherLimiter = promiseLimit(bulkFetcherConcurrency); + + const globalConcurrency = opts.maxConcurrency + ? Math.max(opts.maxConcurrency, 1) + : Math.max(fetcherConcurrency, optionalFetcherConcurrency, bulkFetcherConcurrency); + this.globalLimiter = promiseLimit(globalConcurrency); + this.fetcher = opts.fetcher; + this.optionalFetcher = opts.optionalFetcher; this.bulkFetcher = opts.bulkFetcher; this.onChanged = opts.onChanged; + this.onReset = opts.onReset; + this.internalEventService = services.internalEventService; this.internalEventService.on('quantumCacheUpdated', this.onQuantumCacheUpdated, { // Ignore our own events, otherwise we'll immediately erase any set value. ignoreLocal: true, }); + this.internalEventService.on('quantumCacheReset', this.onQuantumCacheReset, { + // Ignore our own events, otherwise we'll immediately erase any set value. + ignoreLocal: true, + }); + } + + private get callbackMeta(): CallbackMeta { + return { + cache: this, + disposeSignal: this.disposeController.signal, + }; + } + + private get nameForError() { + return `QuantumCache[${this.name}]`; } /** @@ -76,6 +258,14 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { return this.memoryCache.size; } + /** + * Iterates all [key, value] pairs in memory. + * This applies to the local subset view, not the cross-cluster cache state. + */ + [Symbol.iterator](): Iterator<[key: string, value: T]> { + return this.entries(); + } + /** * Iterates all [key, value] pairs in memory. * This applies to the local subset view, not the cross-cluster cache state. @@ -111,11 +301,13 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { /** * Creates or updates a value in the cache, and erases any stale caches across the cluster. - * Fires an onSet event after the cache has been updated in all processes. + * Fires an onChanged event after the cache has been updated in all processes. * Skips if the value is unchanged. */ @bindThis public async set(key: string, value: T): Promise { + this.throwIfDisposed(); + if (this.memoryCache.get(key) === value) { return; } @@ -125,17 +317,19 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] }); if (this.onChanged) { - await this.onChanged([key], this); + await this.onChanged([key], this.callbackMeta); } } /** * Creates or updates multiple value in the cache, and erases any stale caches across the cluster. - * Fires an onSet for each changed item event after the cache has been updated in all processes. + * Fires an onChanged for each changed item event after the cache has been updated in all processes. * Skips if all values are unchanged. */ @bindThis - public async setMany(items: Iterable<[key: string, value: T]>): Promise { + public async setMany(items: Iterable): Promise { + this.throwIfDisposed(); + const changedKeys: string[] = []; for (const item of items) { @@ -149,39 +343,56 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: changedKeys }); if (this.onChanged) { - await this.onChanged(changedKeys, this); + await this.onChanged(changedKeys, this.callbackMeta); } } } /** * Adds a value to the local memory cache without notifying other process. - * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. + * Neither a Redis event nor onChanged callback will be fired, as the value has not actually changed. * This should only be used when the value is known to be current, like after fetching from the database. */ @bindThis public add(key: string, value: T): void { + this.throwIfDisposed(); + this.memoryCache.set(key, value); } /** * Adds multiple values to the local memory cache without notifying other process. - * Neither a Redis event nor onSet callback will be fired, as the value has not actually changed. + * Neither a Redis event nor onChanged callback will be fired, as the value has not actually changed. * This should only be used when the value is known to be current, like after fetching from the database. */ @bindThis - public addMany(items: Iterable<[key: string, value: T]>): void { + public addMany(items: Iterable): void { + this.throwIfDisposed(); + for (const [key, value] of items) { this.memoryCache.set(key, value); } } + /** + * Gets a value from the local memory cache, or throws KeyNotFoundError if not found. + * Returns cached data only - does not make any fetches. + */ + @bindThis + public get(key: string): T { + const result = this.getMaybe(key); + if (result === undefined) { + throw new KeyNotFoundError(this.nameForError, key); + } + return result; + } + /** * Gets a value from the local memory cache, or returns undefined if not found. * Returns cached data only - does not make any fetches. */ @bindThis - public get(key: string): T | undefined { + public getMaybe(key: string): T | undefined { return this.memoryCache.get(key); } @@ -190,44 +401,80 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { * Returns cached data only - does not make any fetches. */ @bindThis - public getMany(keys: Iterable): [key: string, value: T | undefined][] { - const results: [key: string, value: T | undefined][] = []; + public getMany(keys: Iterable): [key: string, value: T][] { + const results: [key: string, value: T][] = []; for (const key of keys) { - results.push([key, this.get(key)]); + const value = this.getMaybe(key); + if (value !== undefined) { + results.push([key, value]); + } } return results; } /** * Gets or fetches a value from the cache. - * Fires an onSet event, but does not emit an update event to other processes. + * Fires an onChanged event, but does not emit an update event to other processes. */ @bindThis public async fetch(key: string): Promise { + this.throwIfDisposed(); + let value = this.memoryCache.get(key); - if (value === undefined) { - value = await this.fetcher(key, this); + if (value == null) { + value = await this.doFetch(key); + this.memoryCache.set(key, value); if (this.onChanged) { - await this.onChanged([key], this); + await this.onChanged([key], this.callbackMeta); } } return value; } /** - * Gets or fetches multiple values from the cache. - * Fires onSet events, but does not emit any update events to other processes. + * Gets or fetches a value from the cache, returning undefined if not found. + * Fires an onChanged event on success, but does not emit an update event to other processes. */ @bindThis - public async fetchMany(keys: Iterable): Promise<[key: string, value: T][]> { + public async fetchMaybe(key: string): Promise { + this.throwIfDisposed(); + + let value = this.memoryCache.get(key); + if (value != null) { + return value; + } + + value = await this.doFetchMaybe(key); + if (value == null) { + return undefined; + } + + this.memoryCache.set(key, value); + + if (this.onChanged) { + await this.onChanged([key], this.callbackMeta); + } + + return value; + } + + /** + * Gets or fetches multiple values from the cache. + * Missing / unmapped values are excluded from the response. + * Fires onChanged event, but does not emit any update events to other processes. + */ + @bindThis + public async fetchMany(keys: Iterable): Promise> { + this.throwIfDisposed(); + const results: [key: string, value: T][] = []; const toFetch: string[] = []; // Spliterate into cached results / uncached keys. for (const key of keys) { - const fromCache = this.get(key); + const fromCache = this.getMaybe(key); if (fromCache) { results.push([key, fromCache]); } else { @@ -237,7 +484,7 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { // Fetch any uncached keys if (toFetch.length > 0) { - const fetched = await this.bulkFetch(toFetch); + const fetched = await this.doFetchMany(toFetch); // Add to cache and return set this.addMany(fetched); @@ -245,11 +492,11 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { // Emit event if (this.onChanged) { - await this.onChanged(toFetch, this); + await this.onChanged(toFetch, this.callbackMeta); } } - return results; + return makeKVPArray(results); } /** @@ -258,30 +505,34 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { */ @bindThis public has(key: string): boolean { - return this.memoryCache.get(key) !== undefined; + return this.memoryCache.has(key); } /** * Deletes a value from the cache, and erases any stale caches across the cluster. - * Fires an onDelete event after the cache has been updated in all processes. + * Fires an onChanged event after the cache has been updated in all processes. */ @bindThis public async delete(key: string): Promise { + this.throwIfDisposed(); + this.memoryCache.delete(key); await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] }); if (this.onChanged) { - await this.onChanged([key], this); + await this.onChanged([key], this.callbackMeta); } } /** * Deletes multiple values from the cache, and erases any stale caches across the cluster. - * Fires an onDelete event for each key after the cache has been updated in all processes. + * Fires an onChanged event for each key after the cache has been updated in all processes. * Skips if the input is empty. */ @bindThis public async deleteMany(keys: Iterable): Promise { + this.throwIfDisposed(); + const deleted: string[] = []; for (const key of keys) { @@ -296,26 +547,56 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: deleted }); if (this.onChanged) { - await this.onChanged(deleted, this); + await this.onChanged(deleted, this.callbackMeta); } } /** * Refreshes the value of a key from the fetcher, and erases any stale caches across the cluster. - * Fires an onSet event after the cache has been updated in all processes. + * Fires an onChanged event after the cache has been updated in all processes. */ @bindThis public async refresh(key: string): Promise { - const value = await this.fetcher(key, this); + this.throwIfDisposed(); + + const value = await this.doFetch(key); await this.set(key, value); return value; } + /** + * Refreshes the value of a key from the fetcher, returning undefined if not found. + * Whether a result is found or not, it then erases any stale caches across the cluster. + * Fires an onChanged event after the cache has been updated in all processes. + */ @bindThis - public async refreshMany(keys: Iterable): Promise<[key: string, value: T][]> { - const values = await this.bulkFetch(keys); - await this.setMany(values); - return values; + public async refreshMaybe(key: string): Promise { + this.throwIfDisposed(); + + const value = await this.doFetchMaybe(key); + + if (value != null) { + await this.set(key, value); + } else { + await this.delete(key); + } + + return value; + } + + /** + * Refreshes multiple values from the cache, and erases any stale caches across the cluster. + * Fires an onChanged event after the cache has been updated in all processes. + * Missing / unmapped values are excluded from the response. + */ + @bindThis + public async refreshMany(keys: Iterable): Promise> { + this.throwIfDisposed(); + + const toFetch = Array.from(keys); + const fetched = await this.doFetchMany(toFetch); + await this.setMany(fetched); + return makeKVPArray(fetched); } /** @@ -323,16 +604,36 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { * Does not send any events or update other processes. */ @bindThis - public clear() { + public clear(): void { + this.throwIfDisposed(); + this.memoryCache.clear(); } + /** + * Erases all entries from the cache. + * Fires an onReset event and updates other processes. + */ + public async reset(): Promise { + this.throwIfDisposed(); + + this.clear(); + + await this.internalEventService.emit('quantumCacheReset', { name: this.name }); + + if (this.onReset) { + await this.onReset(this.callbackMeta); + } + } + /** * Removes expired cache entries from the local view. * Does not send any events or update other processes. */ @bindThis - public gc() { + public gc(): void { + this.throwIfDisposed(); + this.memoryCache.gc(); } @@ -341,45 +642,414 @@ export class QuantumKVCache implements Iterable<[key: string, value: T]> { * This *must* be called when shutting down to prevent memory leaks! */ @bindThis - public dispose() { - this.internalEventService.off('quantumCacheUpdated', this.onQuantumCacheUpdated); + public async dispose(): Promise { + if (this.isDisposed) return; + this.isDisposing = true; - this.memoryCache.dispose(); + try { + // Stop handling events *first* + this.internalEventService.off('quantumCacheUpdated', this.onQuantumCacheUpdated); + this.internalEventService.off('quantumCacheReset', this.onQuantumCacheReset); + + // Kill active fetchers + const error = new DisposingError({ source: this.nameForError }); + this.disposeController.abort(error); + + // Wait for cleanup + await Promise.allSettled([ + ...this.activeFetchers.values().map(p => trackPromise(p)), + ...this.activeOptionalFetchers.values().map(p => trackPromise(p)), + ...this.activeBulkFetchers.values().map(p => trackPromise(p)), + ]); + + // Purge memory for faster GC + this.activeFetchers.clear(); + this.activeOptionalFetchers.clear(); + this.activeBulkFetchers.clear(); + this.memoryCache.dispose(); + } finally { + this.isDisposing = false; + this.isDisposed = true; + } } @bindThis - private async bulkFetch(keys: Iterable): Promise<[key: string, value: T][]> { - if (this.bulkFetcher) { - const results = await this.bulkFetcher(Array.from(keys), this); - return Array.from(results); - } + private async onQuantumCacheUpdated(data: EventTypes['quantumCacheUpdated']): Promise { + this.throwIfDisposed(); - const results: [key: string, value: T][] = []; - for (const key of keys) { - const value = await this.fetcher(key, this); - results.push([key, value]); - } - return results; - } - - @bindThis - private async onQuantumCacheUpdated(data: InternalEventTypes['quantumCacheUpdated']): Promise { if (data.name === this.name) { for (const key of data.keys) { this.memoryCache.delete(key); } if (this.onChanged) { - await this.onChanged(data.keys, this); + await this.onChanged(data.keys, this.callbackMeta); + } + } + } + + @bindThis + private async onQuantumCacheReset(data: EventTypes['quantumCacheReset']): Promise { + this.throwIfDisposed(); + + if (data.name === this.name) { + this.clear(); + + if (this.onReset) { + await this.onReset(this.callbackMeta); } } } /** - * Iterates all [key, value] pairs in memory. - * This applies to the local subset view, not the cross-cluster cache state. + * Executes a fetch operation and translates results. + * Always uses fetcher(). + * Concurrent calls for the same key are de-duplicated. */ - [Symbol.iterator](): Iterator<[key: string, value: T]> { - return this.entries(); + @bindThis + private doFetch(key: string): Promise> { + // De-duplicate call + let promise = this.activeFetchers.get(key); + if (!promise) { + // Start new call + const fetchPromise = promiseTry(this.callFetcher, key) + .catch(async err => { + if (err instanceof EntityNotFoundError) { + throw new KeyNotFoundError(this.nameForError, key, renderInlineError(err), { cause: err }); + } + + throw new FetchFailedError(this.nameForError, key, renderInlineError(err), { cause: err }); + }) + .then(async result => { + if (result == null) { + throw new KeyNotFoundError(this.nameForError, key); + } + return result; + }); + + // Untrack when it finalizes + const cleanupCallback = async () => { + if (this.activeFetchers.get(key) === promise) { + this.activeFetchers.delete(key); + } else { + throw new QuantumCacheError(this.nameForError, `Internal error: fetcher race detected for key "${key}"`); + } + }; + promise = withCleanup(fetchPromise, cleanupCallback); + + // Track it!! + this.activeFetchers.set(key, promise); + } + + return promise; + } + + /** + * Executes a fetchMaybe operation and translates results. + * Automatically uses the best available fetch implementation. + * Concurrent calls for the same key are de-duplicated. + */ + @bindThis + private doFetchMaybe(key: string): Promise | undefined> { + // De-duplicate call + let promise = this.activeOptionalFetchers.get(key); + if (!promise) { + // Use optionalFetcher() if available + if (this.optionalFetcher != null) { + // Start new call + const fetchPromise = promiseTry(this.callOptionalFetcher, key) + .catch(async err => { + throw new FetchFailedError(this.nameForError, key, renderInlineError(err), { cause: err }); + }) + .then(result => result ?? undefined); + + // Untrack when it finalizes + const cleanupCallback = async () => { + if (this.activeOptionalFetchers.get(key) === promise) { + this.activeOptionalFetchers.delete(key); + } else { + throw new QuantumCacheError(this.nameForError, `Internal error: optionalFetcher race detected for key "${key}"`); + } + }; + promise = withCleanup(fetchPromise, cleanupCallback); + + // Track it!! + this.activeOptionalFetchers.set(key, promise); + } else { + // Fall back on fetcher() if optionalFetcher() is unavailable + promise = promiseTry(this.doFetch, key) + .catch(async err => { + if (err instanceof KeyNotFoundError) { + return undefined; + } + + throw err; + }); + } + } + + // Await result + return promise; + } + + /** + * Executes a fetchMany operation and translates results. + * Automatically uses the best available fetch implementation. + * Concurrent calls for the same key are de-duplicated. + */ + @bindThis + private doFetchMany(keys: string[]): Promise<[key: string, value: Value][]> { + const uniqueKeys = new Set(keys); + const fetcherPromises = new Map>(); + const optionalFetcherPromises = new Map>(); + const bulkFetcherPromises = new Set>(); + const remainingKeys: string[] = []; + + // If any keys are covered by an active promise, then re-use it to avoid duplicate fetches. + for (const key of uniqueKeys) { + // Re-use an optionalFetcher() call + const optionalPromise = this.activeOptionalFetchers.get(key); + if (optionalPromise) { + optionalFetcherPromises.set(key, optionalPromise); + continue; + } + + // Re-use a fetcher() call + const fetchPromise = this.activeFetchers.get(key); + if (fetchPromise) { + fetcherPromises.set(key, fetchPromise); + continue; + } + + // Re-use a bulkFetcher() call + const bulkPromise = this.activeBulkFetchers.get(key); + if (bulkPromise) { + bulkFetcherPromises.add(bulkPromise); + continue; + } + + // Queue up for a new bulkFetcher() call + remainingKeys.push(key); + } + + // Start a new fetch for any keys that weren't already covered. + if (hasAtLeastOne(remainingKeys)) { + if (remainingKeys.length > 1 && this.bulkFetcher != null) { + // Use the bulk fetcher if available + const promise = this.callBulkFetcherWithTracking(remainingKeys); + bulkFetcherPromises.add(promise); + } else { + // Otherwise fall back to single fetcher + for (const key of remainingKeys) { + const promise = this.doFetchMaybe(key); + optionalFetcherPromises.set(key, promise); + } + } + } + + return Promise + // Wrap all promises into a common shape + .allSettled[]>([ + ...fetcherPromises + .entries() + .map(([key, promise]) => promise + .catch(async err => { + if (err instanceof KeyNotFoundError) { + return undefined; + } + throw err; + }) + .then(value => { + if (value === undefined) { + return []; + } + return [[key, value]] as KeyValue[]; + })), + ...optionalFetcherPromises + .entries() + .map(([key, promise]) => promise.then(value => { + if (value === undefined) { + return []; + } + return [[key, value]] as KeyValue[]; + })), + ...bulkFetcherPromises, + ]) + // Unpack results and handle errors + .then(async promiseResults => { + const results: KeyValue[][] = []; + const errors: unknown[] = []; + + for (const pr of promiseResults) { + if (pr.status === 'fulfilled') { + results.push(pr.value); + } else { + errors.push(pr.reason); + } + } + + if (errors.length === 1) { + const innerException = errors[0]; + throw new FetchFailedError(this.nameForError, keys, renderInlineError(innerException), { cause: innerException }); + } else if (errors.length > 1) { + const innerException = new AggregateError(errors); + throw new FetchFailedError(this.nameForError, keys, 'Multiple exceptions thrown; see inner exception (cause) for details', { cause: innerException }); + } + + return results.flat(); + }); + } + + /** + * Calls fetcher(). + * Do not call this directly - use doFetch() instead! + */ + @bindThis + private callFetcher(key: string): Promise { + // Safety check, in case this gets called directly by mistake + this.throwIfDisposed(); + if (this.activeFetchers.has(key)) { + throw new QuantumCacheError(this.nameForError, `Internal error: attempted to call fetcher multiple times for key "${key}"`); + } + + // Start limiter cascade + return this.globalLimiter(async () => { + this.throwIfDisposed(); + + return await this.fetcherLimiter(async () => { + this.throwIfDisposed(); + + return await withSignal( + // Execute callback and adapt results + async () => await this.fetcher(key, this.callbackMeta), + + // Bind abort signal in case fetcher stalls out + this.disposeController.signal, + ); + }); + }); + } + + /** + * Calls optionalFetcher(). + * Do not call this directly - use doFetchMaybe() instead! + */ + @bindThis + private callOptionalFetcher(key: string): Promise { + // Safety checks, in case this gets called directly by mistake + this.throwIfDisposed(); + const optionalFetcher = this.optionalFetcher; + if (optionalFetcher == null) { + throw new QuantumCacheError(this.nameForError, 'Internal error: attempted to call optionalFetcher for a cache that doesn\'t support it'); + } + if (this.activeOptionalFetchers.has(key)) { + throw new QuantumCacheError(this.nameForError, `Internal error: attempted to call optionalFetcher multiple times for key "${key}"`); + } + + // Start limiter cascade + return this.globalLimiter(async () => { + this.throwIfDisposed(); + + return await this.optionalFetcherLimiter(async () => { + this.throwIfDisposed(); + + return await withSignal( + // Execute callback and adapt results + async () => await optionalFetcher(key, this.callbackMeta), + + // Bind abort signal in case fetcher stalls out + this.disposeController.signal, + ); + }); + }); + } + + /** + * Calls bulkFetcher() and tracks the promise. + * Do not call this directly - use doBulkFetch() instead! + */ + @bindThis + private callBulkFetcherWithTracking(keys: AtLeastOne): ActiveBulkFetcher { + // Start new call + const fetchPromise = promiseTry(this.callBulkFetcher, keys) + .then(results => Array.from(results).filter((result): result is KeyValue => { + return result[1] != null; + })); + + // Untrack when it finalizes + const cleanupCallback = async () => { + const racedKeys: string[] = []; + + for (const key of keys) { + if (this.activeBulkFetchers.get(key) === promise) { + this.activeBulkFetchers.delete(key); + } else { + racedKeys.push(key); + } + } + + if (racedKeys.length > 0) { + const allKeys = racedKeys.map(k => `"${k}"`).join(', '); + throw new QuantumCacheError(this.nameForError, `Internal error: bulkFetcher race detected for key(s) ${allKeys}`); + } + }; + const promise = withCleanup(fetchPromise, cleanupCallback); + + // Track it!! + for (const key of keys) { + this.activeBulkFetchers.set(key, promise); + } + + return promise; + } + + /** + * Calls bulkFetcher(). + * Do not call this directly - use bulkFetch() instead! + */ + @bindThis + private callBulkFetcher(keys: AtLeastOne): Promise>> { + // Safety checks, in case this gets called directly by mistake + const bulkFetcher = this.bulkFetcher; + this.throwIfDisposed(); + if (bulkFetcher == null) { + throw new QuantumCacheError(this.nameForError, 'Internal error: attempted to call bulkFetcher for a cache that doesn\'t support it'); + } + const duplicateKeys = keys.filter(key => this.activeBulkFetchers.has(key)); + if (duplicateKeys.length > 0) { + const allKeys = duplicateKeys.map(k => `"${k}"`).join(', '); + throw new QuantumCacheError(this.nameForError, `Internal error: attempted to call bulkFetcher multiple times for key(s) ${allKeys}`); + } + + // Start limiter cascade + return this.globalLimiter(async () => { + this.throwIfDisposed(); + + return await this.bulkFetcherLimiter(async () => { + this.throwIfDisposed(); + + return await withSignal( + // Execute callback and adapt results + async () => await bulkFetcher(keys, this.callbackMeta), + + // Bind abort signal in case fetcher stalls out + this.disposeController.signal, + ); + }); + }); + } + + @bindThis + private throwIfDisposed() { + if (this.isDisposing) { + throw new DisposingError({ source: this.nameForError }); + } + if (this.isDisposed) { + throw new DisposedError({ source: this.nameForError }); + } } } + +function hasAtLeastOne(array: T[]): array is AtLeastOne { + return array.length > 0; +} diff --git a/packages/backend/src/misc/bigint.ts b/packages/backend/src/misc/bigint.ts index efa1527ec9..58603400b2 100644 --- a/packages/backend/src/misc/bigint.ts +++ b/packages/backend/src/misc/bigint.ts @@ -4,7 +4,7 @@ */ function parseBigIntChunked(str: string, base: number, chunkSize: number, powerOfChunkSize: bigint): bigint { - const chunks = []; + const chunks: string[] = []; while (str.length > 0) { chunks.unshift(str.slice(-chunkSize)); str = str.slice(0, -chunkSize); diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 666e684c1c..a5b36d923f 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -3,10 +3,24 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as Redis from 'ioredis'; +import type * as Redis from 'ioredis'; import { bindThis } from '@/decorators.js'; +import type { TimeService } from '@/global/TimeService.js'; + +export interface RedisCacheServices extends MemoryCacheServices { + readonly redisClient: Redis.Redis +} + +export interface RedisKVCacheOpts { + lifetime: number; + memoryCacheLifetime: number; + fetcher?: RedisKVCache['fetcher']; + toRedisConverter?: RedisKVCache['toRedisConverter']; + fromRedisConverter?: RedisKVCache['fromRedisConverter']; +} export class RedisKVCache { + private readonly redisClient: Redis.Redis; private readonly lifetime: number; private readonly memoryCache: MemoryKVCache; public readonly fetcher: (key: string) => Promise; @@ -14,18 +28,15 @@ export class RedisKVCache { public readonly fromRedisConverter: (value: string) => T | undefined; constructor( - private redisClient: Redis.Redis, - private name: string, - opts: { - lifetime: RedisKVCache['lifetime']; - memoryCacheLifetime: number; - fetcher?: RedisKVCache['fetcher']; - toRedisConverter?: RedisKVCache['toRedisConverter']; - fromRedisConverter?: RedisKVCache['fromRedisConverter']; - }, + public name: string, + services: RedisCacheServices, + opts: RedisKVCacheOpts, ) { + this.redisClient = services.redisClient; this.lifetime = opts.lifetime; - this.memoryCache = new MemoryKVCache(opts.memoryCacheLifetime); + // OK: we forward all management calls to the inner cache. + // eslint-disable-next-line no-restricted-syntax + this.memoryCache = new MemoryKVCache(name + ':mem', services, { lifetime: opts.memoryCacheLifetime }); this.fetcher = opts.fetcher ?? (() => { throw new Error('fetch not supported - use get/set directly'); }); this.toRedisConverter = opts.toRedisConverter ?? ((value) => JSON.stringify(value)); this.fromRedisConverter = opts.fromRedisConverter ?? ((value) => JSON.parse(value)); @@ -39,7 +50,7 @@ export class RedisKVCache { `kvcache:${this.name}:${key}`, this.toRedisConverter(value), ); - } else { + } else if (this.lifetime > 0) { await this.redisClient.set( `kvcache:${this.name}:${key}`, this.toRedisConverter(value), @@ -115,7 +126,16 @@ export class RedisKVCache { } } +export interface RedisSingleCacheOpts { + lifetime: number; + memoryCacheLifetime: number; + fetcher?: RedisSingleCache['fetcher']; + toRedisConverter?: RedisSingleCache['toRedisConverter']; + fromRedisConverter?: RedisSingleCache['fromRedisConverter']; +} + export class RedisSingleCache { + private readonly redisClient: Redis.Redis; private readonly lifetime: number; private readonly memoryCache: MemorySingleCache; public readonly fetcher: () => Promise; @@ -123,18 +143,15 @@ export class RedisSingleCache { public readonly fromRedisConverter: (value: string) => T | undefined; constructor( - private redisClient: Redis.Redis, - private name: string, - opts: { - lifetime: number; - memoryCacheLifetime: number; - fetcher?: RedisSingleCache['fetcher']; - toRedisConverter?: RedisSingleCache['toRedisConverter']; - fromRedisConverter?: RedisSingleCache['fromRedisConverter']; - }, + public name: string, + services: RedisCacheServices, + opts: RedisSingleCacheOpts, ) { + this.redisClient = services.redisClient; this.lifetime = opts.lifetime; - this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime); + // OK: we forward all management calls to the inner cache. + // eslint-disable-next-line no-restricted-syntax + this.memoryCache = new MemorySingleCache(name + ':mem', services, { lifetime: opts.memoryCacheLifetime }); this.fetcher = opts.fetcher ?? (() => { throw new Error('fetch not supported - use get/set directly'); }); this.toRedisConverter = opts.toRedisConverter ?? ((value) => JSON.stringify(value)); @@ -149,7 +166,7 @@ export class RedisSingleCache { `singlecache:${this.name}`, this.toRedisConverter(value), ); - } else { + } else if (this.lifetime > 0) { await this.redisClient.set( `singlecache:${this.name}`, this.toRedisConverter(value), @@ -174,12 +191,22 @@ export class RedisSingleCache { return value; } + @bindThis + public gc(): void { + this.memoryCache.gc(); + } + @bindThis public async delete(): Promise { this.memoryCache.delete(); await this.redisClient.del(`singlecache:${this.name}`); } + @bindThis + public clear(): void { + this.memoryCache.clear(); + } + /** * キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します * This awaits the call to Redis to ensure that the write succeeded, which is important for a few reasons: @@ -208,17 +235,37 @@ export class RedisSingleCache { // TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする } + + @bindThis + public dispose(): void { + this.clear(); + this.memoryCache.dispose(); + } +} + +export interface MemoryCacheServices { + readonly timeService: TimeService; +} + +export interface MemoryCacheOpts { + lifetime: number; } // TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする？ export class MemoryKVCache { private readonly cache = new Map(); - private readonly gcIntervalHandle = setInterval(() => this.gc(), 1000 * 60 * 3); // 3m + private readonly timeService: TimeService; + private readonly lifetime: number; constructor( - private readonly lifetime: number, - ) {} + public readonly name: string, + services: MemoryCacheServices, + opts: MemoryCacheOpts, + ) { + this.timeService = services.timeService; + this.lifetime = opts.lifetime; + } @bindThis /** @@ -227,7 +274,7 @@ export class MemoryKVCache { */ public set(key: string, value: T): void { this.cache.set(key, { - date: Date.now(), + date: this.timeService.now, value, }); } @@ -236,7 +283,7 @@ export class MemoryKVCache { public get(key: string): T | undefined { const cached = this.cache.get(key); if (cached == null) return undefined; - if ((Date.now() - cached.date) > this.lifetime) { + if ((this.timeService.now - cached.date) > this.lifetime) { this.cache.delete(key); return undefined; } @@ -246,7 +293,7 @@ export class MemoryKVCache { public has(key: string): boolean { const cached = this.cache.get(key); if (cached == null) return false; - if ((Date.now() - cached.date) > this.lifetime) { + if ((this.timeService.now - cached.date) > this.lifetime) { this.cache.delete(key); return false; } @@ -312,7 +359,7 @@ export class MemoryKVCache { @bindThis public gc(): void { - const now = Date.now(); + const now = this.timeService.now; for (const [key, { date }] of this.cache.entries()) { // The map is ordered from oldest to youngest. @@ -335,7 +382,6 @@ export class MemoryKVCache { @bindThis public dispose(): void { this.clear(); - clearInterval(this.gcIntervalHandle); } public get size() { @@ -348,27 +394,46 @@ export class MemoryKVCache { } export class MemorySingleCache { + private readonly timeService: TimeService; + private readonly lifetime: number; + private cachedAt: number | null = null; private value: T | undefined; constructor( - private lifetime: number, - ) {} + public readonly name: string, + services: MemoryCacheServices, + opts: MemoryCacheOpts, + ) { + this.timeService = services.timeService; + this.lifetime = opts.lifetime; + } @bindThis public set(value: T): void { - this.cachedAt = Date.now(); + this.cachedAt = this.timeService.now; this.value = value; } @bindThis - public get(): T | undefined { - if (this.cachedAt == null) return undefined; - if ((Date.now() - this.cachedAt) > this.lifetime) { - this.value = undefined; - this.cachedAt = null; - return undefined; + public gc(): void { + // Check if we have a valid, non-expired value. + // This is a little convoluted but protects against edge cases and invalid states. + if (this.value !== undefined && this.cachedAt != null) { + const age = this.timeService.now - this.cachedAt; + if (Number.isSafeInteger(age) && age <= this.lifetime) { + return; + } } + + // If we get here, then it's expired or otherwise invalid. + // Whatever the case, we should clear everything back to zeros. + this.delete(); + } + + @bindThis + public get(): T | undefined { + this.gc(); return this.value; } @@ -378,6 +443,11 @@ export class MemorySingleCache { this.cachedAt = null; } + @bindThis + public clear() { + this.delete(); + } + /** * キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します * optional: キャッシュが存在してもvalidatorでfalseを返すとキャッシュ無効扱いにします @@ -429,4 +499,9 @@ export class MemorySingleCache { } return value; } + + @bindThis + public dispose() { + this.clear(); + } } diff --git a/packages/backend/src/misc/call-all.ts b/packages/backend/src/misc/call-all.ts new file mode 100644 index 0000000000..08e23d34ab --- /dev/null +++ b/packages/backend/src/misc/call-all.ts @@ -0,0 +1,117 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { promiseTry } from '@/misc/promise-try.js'; + +/** + * Calls a group of synchronous functions with the given parameters. + * Errors are suppressed and aggregated, ensuring that nothing is thrown until all calls have completed. + * This ensures that an error in one callback does not prevent later callbacks from completing. + * @param funcs Callback functions to execute + * @param args Arguments to pass to each callback + */ +export function callAll(funcs: Iterable<(...args: T) => void>, ...args: T): void { + const errors: unknown[] = []; + + for (const func of funcs) { + try { + func(...args); + } catch (err) { + errors.push(err); + } + } + + if (errors.length > 0) { + throw new AggregateError(errors); + } +} + +/** + * Calls a group of async functions with the given parameters. + * Errors are suppressed and aggregated, ensuring that nothing is thrown until all calls have completed. + * This ensures that an error in one callback does not prevent later callbacks from completing. + * Callbacks are executed in parallel using Promise.allSettled(). + * @param funcs Callback functions to execute + * @param args Arguments to pass to each callback + */ +export async function callAllAsync(funcs: Iterable<(...args: T) => Promise | void>, ...args: T): Promise { + // Start all the tasks + const promises = Array.from(funcs) + .map(func => { + // Handle errors thrown synchronously + return promiseTry(() => func(...args)); + }); + + // Wait for all to finish + const results = await Promise.allSettled(promises); + + // Check for errors + const errors = results.filter(r => r.status === 'rejected').map(r => r.reason as unknown); + if (errors.length > 0) { + throw new AggregateError(errors); + } +} + +/** + * Calls a single synchronous method across a group of object, passing the given parameters as values. + * Errors are suppressed and aggregated, ensuring that nothing is thrown until all calls have completed. + * This ensures that an error in one callback does not prevent later callbacks from completing. + * @param objects Objects to execute methods on + * @param method Name (property key) of the method to execute + * @param args Arguments to pass + */ +export function callAllOn>(objects: Iterable, method: TMethod, ...args: MethodParams): void { + const errors: unknown[] = []; + + for (const object of objects) { + try { + // @ts-expect-error Our generic constraints ensure this is safe, but TS can't infer that much context. + object[method](...args); + } catch (err) { + errors.push(err); + } + } + + if (errors.length > 0) { + throw new AggregateError(errors); + } +} + +/** + * Calls a single asynchronous method across a group of object, passing the given parameters as values. + * Errors are suppressed and aggregated, ensuring that nothing is thrown until all calls have completed. + * This ensures that an error in one callback does not prevent later callbacks from completing. + * Callbacks are executed in parallel using Promise.allSettled(). + * @param objects Objects to execute methods on + * @param method Name (property key) of the method to execute + * @param args Arguments to pass + */ +export async function callAllOnAsync>(objects: Iterable, method: TMethod, ...args: MethodParams): Promise { + // Start all the tasks + const promises = Array.from(objects) + .map(object => { + // Handle errors thrown synchronously + return promiseTry(() => { + // @ts-expect-error Our generic constraints ensure this is safe, but TS can't infer that much context. + return object[method](...args); + }); + }); + + // Wait for all to finish + const results = await Promise.allSettled(promises); + + // Check for errors + const errors = results.filter(r => r.status === 'rejected').map(r => r.reason as unknown); + if (errors.length > 0) { + throw new AggregateError(errors); + } +} + +type AnyFunc = (...args: unknown[]) => unknown; +type Methods = { + [Key in keyof TObject]: TObject[Key] extends AnyFunc ? TObject[Key] : never; +}; +type MethodKeys = keyof Methods; +type MethodParams> = TObject[TMethod] extends AnyFunc ? Parameters : never; diff --git a/packages/backend/src/misc/captcha-error.ts b/packages/backend/src/misc/captcha-error.ts index 217018ec68..9e91826c81 100644 --- a/packages/backend/src/misc/captcha-error.ts +++ b/packages/backend/src/misc/captcha-error.ts @@ -3,9 +3,20 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import type { CaptchaErrorCode } from '@/core/CaptchaService.js'; +export const captchaErrorCodes = { + invalidProvider: Symbol('invalidProvider'), + invalidParameters: Symbol('invalidParameters'), + noResponseProvided: Symbol('noResponseProvided'), + requestFailed: Symbol('requestFailed'), + verificationFailed: Symbol('verificationFailed'), + unknown: Symbol('unknown'), +} as const; +export type CaptchaErrorCode = typeof captchaErrorCodes[keyof typeof captchaErrorCodes]; export class CaptchaError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + public readonly code: CaptchaErrorCode; public readonly cause?: unknown; diff --git a/packages/backend/src/misc/collapsed-queue.ts b/packages/backend/src/misc/collapsed-queue.ts index 5bc20a78ae..168a4d7680 100644 --- a/packages/backend/src/misc/collapsed-queue.ts +++ b/packages/backend/src/misc/collapsed-queue.ts @@ -3,9 +3,11 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import type { TimeService, TimerHandle } from '@/global/TimeService.js'; + type Job = { value: V; - timer: NodeJS.Timeout; + timer: TimerHandle; }; // TODO: redis使えるようにする @@ -13,6 +15,7 @@ export class CollapsedQueue { private jobs: Map> = new Map(); constructor( + protected readonly timeService: TimeService, private timeout: number, private collapse: (oldValue: V, newValue: V) => V, private perform: (key: K, value: V) => Promise, @@ -24,7 +27,7 @@ export class CollapsedQueue { const merged = this.collapse(old.value, value); this.jobs.set(key, { ...old, value: merged }); } else { - const timer = setTimeout(() => { + const timer = this.timeService.startTimer(() => { const job = this.jobs.get(key)!; this.jobs.delete(key); this.perform(key, job.value); @@ -37,7 +40,7 @@ export class CollapsedQueue { const entries = [...this.jobs.entries()]; this.jobs.clear(); for (const [_key, job] of entries) { - clearTimeout(job.timer); + this.timeService.stopTimer(job.timer); } await Promise.allSettled(entries.map(([key, job]) => this.perform(key, job.value))); } diff --git a/packages/backend/src/misc/emoji-regex.ts b/packages/backend/src/misc/emoji-regex.ts index 53e66298a6..a3575d778c 100644 --- a/packages/backend/src/misc/emoji-regex.ts +++ b/packages/backend/src/misc/emoji-regex.ts @@ -4,7 +4,7 @@ */ // taken from @twemoji/parser/dist/lib/regex.js -const twemojiRegex = /(?:\ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83e\uddd1\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83e\uddd1\ud83c[\udffc-\udfff]|\ud83e\uddd1\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83e\uddd1\ud83c[\udffb\udffd-\udfff]|\ud83e\uddd1\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83e\uddd1\ud83c[\udffb\udffc\udffe\udfff]|\ud83e\uddd1\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83e\uddd1\ud83c[\udffb-\udffd\udfff]|\ud83e\uddd1\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83e\uddd1\ud83c[\udffb-\udffe]|\ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffc-\udfff]|\ud83d\udc68\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffc\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb\udffd-\udfff]|\ud83d\udc68\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffd\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb\udffc\udffe\udfff]|\ud83d\udc68\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffe\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb-\udffd\udfff]|\ud83d\udc68\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udfff\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb-\udffe]|\ud83d\udc69\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffc-\udfff]|\ud83d\udc69\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83d\udc69\ud83c[\udffc-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb\udffd-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\ud83e\udd1d\u200d\ud83d\udc69\ud83c[\udffb\udffd-\udfff]|\ud83d\udc69\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffd\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb\udffc\udffe\udfff]|\ud83d\udc69\ud83c\udffd\u200d\ud83e\udd1d\u200d\ud83d\udc69\ud83c[\udffb\udffc\udffe\udfff]|\ud83d\udc69\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffe\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb-\udffd\udfff]|\ud83d\udc69\ud83c\udffe\u200d\ud83e\udd1d\u200d\ud83d\udc69\ud83c[\udffb-\udffd\udfff]|\ud83d\udc69\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udfff\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb-\udffe]|\ud83d\udc69\ud83c\udfff\u200d\ud83e\udd1d\u200d\ud83d\udc69\ud83c[\udffb-\udffe]|\ud83e\uddd1\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83e\uddd1\ud83c[\udffc-\udfff]|\ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c[\udffb-\udfff]|\ud83e\uddd1\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83e\uddd1\ud83c[\udffb\udffd-\udfff]|\ud83e\uddd1\ud83c\udffc\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c[\udffb-\udfff]|\ud83e\uddd1\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83e\uddd1\ud83c[\udffb\udffc\udffe\udfff]|\ud83e\uddd1\ud83c\udffd\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c[\udffb-\udfff]|\ud83e\uddd1\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83e\uddd1\ud83c[\udffb-\udffd\udfff]|\ud83e\uddd1\ud83c\udffe\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c[\udffb-\udfff]|\ud83e\uddd1\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83e\uddd1\ud83c[\udffb-\udffe]|\ud83e\uddd1\ud83c\udfff\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c[\udffb-\udfff]|\ud83d\udc68\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68|\ud83d\udc69\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d[\udc68\udc69]|\ud83e\udef1\ud83c\udffb\u200d\ud83e\udef2\ud83c[\udffc-\udfff]|\ud83e\udef1\ud83c\udffc\u200d\ud83e\udef2\ud83c[\udffb\udffd-\udfff]|\ud83e\udef1\ud83c\udffd\u200d\ud83e\udef2\ud83c[\udffb\udffc\udffe\udfff]|\ud83e\udef1\ud83c\udffe\u200d\ud83e\udef2\ud83c[\udffb-\udffd\udfff]|\ud83e\udef1\ud83c\udfff\u200d\ud83e\udef2\ud83c[\udffb-\udffe]|\ud83d\udc68\u200d\u2764\ufe0f\u200d\ud83d\udc68|\ud83d\udc69\u200d\u2764\ufe0f\u200d\ud83d[\udc68\udc69]|\ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1|\ud83d\udc6b\ud83c[\udffb-\udfff]|\ud83d\udc6c\ud83c[\udffb-\udfff]|\ud83d\udc6d\ud83c[\udffb-\udfff]|\ud83d\udc8f\ud83c[\udffb-\udfff]|\ud83d\udc91\ud83c[\udffb-\udfff]|\ud83e\udd1d\ud83c[\udffb-\udfff]|\ud83d[\udc6b-\udc6d\udc8f\udc91]|\ud83e\udd1d)|(?:\ud83d[\udc68\udc69]|\ud83e\uddd1)(?:\ud83c[\udffb-\udfff])?\u200d(?:\u2695\ufe0f|\u2696\ufe0f|\u2708\ufe0f|\ud83c[\udf3e\udf73\udf7c\udf84\udf93\udfa4\udfa8\udfeb\udfed]|\ud83d[\udcbb\udcbc\udd27\udd2c\ude80\ude92]|\ud83e[\uddaf-\uddb3\uddbc\uddbd])|(?:\ud83c[\udfcb\udfcc]|\ud83d[\udd74\udd75]|\u26f9)((?:\ud83c[\udffb-\udfff]|\ufe0f)\u200d[\u2640\u2642]\ufe0f)|(?:\ud83c[\udfc3\udfc4\udfca]|\ud83d[\udc6e\udc70\udc71\udc73\udc77\udc81\udc82\udc86\udc87\ude45-\ude47\ude4b\ude4d\ude4e\udea3\udeb4-\udeb6]|\ud83e[\udd26\udd35\udd37-\udd39\udd3d\udd3e\uddb8\uddb9\uddcd-\uddcf\uddd4\uddd6-\udddd])(?:\ud83c[\udffb-\udfff])?\u200d[\u2640\u2642]\ufe0f|(?:\ud83d\udc68\u200d\ud83d\udc68\u200d\ud83d\udc66\u200d\ud83d\udc66|\ud83d\udc68\u200d\ud83d\udc68\u200d\ud83d\udc67\u200d\ud83d[\udc66\udc67]|\ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d\udc66\u200d\ud83d\udc66|\ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d[\udc66\udc67]|\ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc66\u200d\ud83d\udc66|\ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d[\udc66\udc67]|\ud83d\udc68\u200d\ud83d\udc66\u200d\ud83d\udc66|\ud83d\udc68\u200d\ud83d\udc67\u200d\ud83d[\udc66\udc67]|\ud83d\udc68\u200d\ud83d\udc68\u200d\ud83d[\udc66\udc67]|\ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d[\udc66\udc67]|\ud83d\udc69\u200d\ud83d\udc66\u200d\ud83d\udc66|\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d[\udc66\udc67]|\ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d[\udc66\udc67]|\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f|\ud83c\udff3\ufe0f\u200d\ud83c\udf08|\ud83d\ude36\u200d\ud83c\udf2b\ufe0f|\u2764\ufe0f\u200d\ud83d\udd25|\u2764\ufe0f\u200d\ud83e\ude79|\ud83c\udff4\u200d\u2620\ufe0f|\ud83d\udc15\u200d\ud83e\uddba|\ud83d\udc3b\u200d\u2744\ufe0f|\ud83d\udc41\u200d\ud83d\udde8|\ud83d\udc68\u200d\ud83d[\udc66\udc67]|\ud83d\udc69\u200d\ud83d[\udc66\udc67]|\ud83d\udc6f\u200d\u2640\ufe0f|\ud83d\udc6f\u200d\u2642\ufe0f|\ud83d\ude2e\u200d\ud83d\udca8|\ud83d\ude35\u200d\ud83d\udcab|\ud83e\udd3c\u200d\u2640\ufe0f|\ud83e\udd3c\u200d\u2642\ufe0f|\ud83e\uddde\u200d\u2640\ufe0f|\ud83e\uddde\u200d\u2642\ufe0f|\ud83e\udddf\u200d\u2640\ufe0f|\ud83e\udddf\u200d\u2642\ufe0f|\ud83d\udc08\u200d\u2b1b|\ud83d\udc26\u200d\u2b1b)|[#*0-9]\ufe0f?\u20e3|(?:[©®\u2122\u265f]\ufe0f)|(?:\ud83c[\udc04\udd70\udd71\udd7e\udd7f\ude02\ude1a\ude2f\ude37\udf21\udf24-\udf2c\udf36\udf7d\udf96\udf97\udf99-\udf9b\udf9e\udf9f\udfcd\udfce\udfd4-\udfdf\udff3\udff5\udff7]|\ud83d[\udc3f\udc41\udcfd\udd49\udd4a\udd6f\udd70\udd73\udd76-\udd79\udd87\udd8a-\udd8d\udda5\udda8\uddb1\uddb2\uddbc\uddc2-\uddc4\uddd1-\uddd3\udddc-\uddde\udde1\udde3\udde8\uddef\uddf3\uddfa\udecb\udecd-\udecf\udee0-\udee5\udee9\udef0\udef3]|[\u203c\u2049\u2139\u2194-\u2199\u21a9\u21aa\u231a\u231b\u2328\u23cf\u23ed-\u23ef\u23f1\u23f2\u23f8-\u23fa\u24c2\u25aa\u25ab\u25b6\u25c0\u25fb-\u25fe\u2600-\u2604\u260e\u2611\u2614\u2615\u2618\u2620\u2622\u2623\u2626\u262a\u262e\u262f\u2638-\u263a\u2640\u2642\u2648-\u2653\u2660\u2663\u2665\u2666\u2668\u267b\u267f\u2692-\u2697\u2699\u269b\u269c\u26a0\u26a1\u26a7\u26aa\u26ab\u26b0\u26b1\u26bd\u26be\u26c4\u26c5\u26c8\u26cf\u26d1\u26d3\u26d4\u26e9\u26ea\u26f0-\u26f5\u26f8\u26fa\u26fd\u2702\u2708\u2709\u270f\u2712\u2714\u2716\u271d\u2721\u2733\u2734\u2744\u2747\u2757\u2763\u2764\u27a1\u2934\u2935\u2b05-\u2b07\u2b1b\u2b1c\u2b50\u2b55\u3030\u303d\u3297\u3299])(?:\ufe0f|(?!\ufe0e))|(?:(?:\ud83c[\udfcb\udfcc]|\ud83d[\udd74\udd75\udd90]|\ud83e\udef0|[\u261d\u26f7\u26f9\u270c\u270d])(?:\ufe0f|(?!\ufe0e))|(?:\ud83c[\udf85\udfc2-\udfc4\udfc7\udfca]|\ud83d[\udc42\udc43\udc46-\udc50\udc66-\udc69\udc6e\udc70-\udc78\udc7c\udc81-\udc83\udc85-\udc87\udcaa\udd7a\udd95\udd96\ude45-\ude47\ude4b-\ude4f\udea3\udeb4-\udeb6\udec0\udecc]|\ud83e[\udd0c\udd0f\udd18-\udd1c\udd1e\udd1f\udd26\udd30-\udd39\udd3d\udd3e\udd77\uddb5\uddb6\uddb8\uddb9\uddbb\uddcd-\uddcf\uddd1-\udddd\udec3-\udec5\udef1-\udef8]|[\u270a\u270b]))(?:\ud83c[\udffb-\udfff])?|(?:\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f|\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc73\udb40\udc63\udb40\udc74\udb40\udc7f|\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc77\udb40\udc6c\udb40\udc73\udb40\udc7f|\ud83c\udde6\ud83c[\udde8-\uddec\uddee\uddf1\uddf2\uddf4\uddf6-\uddfa\uddfc\uddfd\uddff]|\ud83c\udde7\ud83c[\udde6\udde7\udde9-\uddef\uddf1-\uddf4\uddf6-\uddf9\uddfb\uddfc\uddfe\uddff]|\ud83c\udde8\ud83c[\udde6\udde8\udde9\uddeb-\uddee\uddf0-\uddf5\uddf7\uddfa-\uddff]|\ud83c\udde9\ud83c[\uddea\uddec\uddef\uddf0\uddf2\uddf4\uddff]|\ud83c\uddea\ud83c[\udde6\udde8\uddea\uddec\udded\uddf7-\uddfa]|\ud83c\uddeb\ud83c[\uddee-\uddf0\uddf2\uddf4\uddf7]|\ud83c\uddec\ud83c[\udde6\udde7\udde9-\uddee\uddf1-\uddf3\uddf5-\uddfa\uddfc\uddfe]|\ud83c\udded\ud83c[\uddf0\uddf2\uddf3\uddf7\uddf9\uddfa]|\ud83c\uddee\ud83c[\udde8-\uddea\uddf1-\uddf4\uddf6-\uddf9]|\ud83c\uddef\ud83c[\uddea\uddf2\uddf4\uddf5]|\ud83c\uddf0\ud83c[\uddea\uddec-\uddee\uddf2\uddf3\uddf5\uddf7\uddfc\uddfe\uddff]|\ud83c\uddf1\ud83c[\udde6-\udde8\uddee\uddf0\uddf7-\uddfb\uddfe]|\ud83c\uddf2\ud83c[\udde6\udde8-\udded\uddf0-\uddff]|\ud83c\uddf3\ud83c[\udde6\udde8\uddea-\uddec\uddee\uddf1\uddf4\uddf5\uddf7\uddfa\uddff]|\ud83c\uddf4\ud83c\uddf2|\ud83c\uddf5\ud83c[\udde6\uddea-\udded\uddf0-\uddf3\uddf7-\uddf9\uddfc\uddfe]|\ud83c\uddf6\ud83c\udde6|\ud83c\uddf7\ud83c[\uddea\uddf4\uddf8\uddfa\uddfc]|\ud83c\uddf8\ud83c[\udde6-\uddea\uddec-\uddf4\uddf7-\uddf9\uddfb\uddfd-\uddff]|\ud83c\uddf9\ud83c[\udde6\udde8\udde9\uddeb-\udded\uddef-\uddf4\uddf7\uddf9\uddfb\uddfc\uddff]|\ud83c\uddfa\ud83c[\udde6\uddec\uddf2\uddf3\uddf8\uddfe\uddff]|\ud83c\uddfb\ud83c[\udde6\udde8\uddea\uddec\uddee\uddf3\uddfa]|\ud83c\uddfc\ud83c[\uddeb\uddf8]|\ud83c\uddfd\ud83c\uddf0|\ud83c\uddfe\ud83c[\uddea\uddf9]|\ud83c\uddff\ud83c[\udde6\uddf2\uddfc]|\ud83c[\udccf\udd8e\udd91-\udd9a\udde6-\uddff\ude01\ude32-\ude36\ude38-\ude3a\ude50\ude51\udf00-\udf20\udf2d-\udf35\udf37-\udf7c\udf7e-\udf84\udf86-\udf93\udfa0-\udfc1\udfc5\udfc6\udfc8\udfc9\udfcf-\udfd3\udfe0-\udff0\udff4\udff8-\udfff]|\ud83d[\udc00-\udc3e\udc40\udc44\udc45\udc51-\udc65\udc6a\udc6f\udc79-\udc7b\udc7d-\udc80\udc84\udc88-\udc8e\udc90\udc92-\udca9\udcab-\udcfc\udcff-\udd3d\udd4b-\udd4e\udd50-\udd67\udda4\uddfb-\ude44\ude48-\ude4a\ude80-\udea2\udea4-\udeb3\udeb7-\udebf\udec1-\udec5\uded0-\uded2\uded5-\uded7\udedc-\udedf\udeeb\udeec\udef4-\udefc\udfe0-\udfeb\udff0]|\ud83e[\udd0d\udd0e\udd10-\udd17\udd20-\udd25\udd27-\udd2f\udd3a\udd3c\udd3f-\udd45\udd47-\udd76\udd78-\uddb4\uddb7\uddba\uddbc-\uddcc\uddd0\uddde-\uddff\ude70-\ude7c\ude80-\ude88\ude90-\udebd\udebf-\udec2\udece-\udedb\udee0-\udee8]|[\u23e9-\u23ec\u23f0\u23f3\u267e\u26ce\u2705\u2728\u274c\u274e\u2753-\u2755\u2795-\u2797\u27b0\u27bf\ue50a])|\ufe0f/g; +const twemojiRegex = /(?:\ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83e\uddd1\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83e\uddd1\ud83c[\udffc-\udfff]|\ud83e\uddd1\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83e\uddd1\ud83c[\udffb\udffd-\udfff]|\ud83e\uddd1\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83e\uddd1\ud83c[\udffb\udffc\udffe\udfff]|\ud83e\uddd1\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83e\uddd1\ud83c[\udffb-\udffd\udfff]|\ud83e\uddd1\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83e\uddd1\ud83c[\udffb-\udffe]|\ud83d\udc68\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffc-\udfff]|\ud83d\udc68\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffc\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb\udffd-\udfff]|\ud83d\udc68\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffd\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb\udffc\udffe\udfff]|\ud83d\udc68\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udffe\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb-\udffd\udfff]|\ud83d\udc68\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc68\ud83c\udfff\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb-\udffe]|\ud83d\udc69\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffc-\udfff]|\ud83d\udc69\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83d\udc69\ud83c[\udffc-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb\udffd-\udfff]|\ud83d\udc69\ud83c\udffc\u200d\ud83e\udd1d\u200d\ud83d\udc69\ud83c[\udffb\udffd-\udfff]|\ud83d\udc69\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffd\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb\udffc\udffe\udfff]|\ud83d\udc69\ud83c\udffd\u200d\ud83e\udd1d\u200d\ud83d\udc69\ud83c[\udffb\udffc\udffe\udfff]|\ud83d\udc69\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udffe\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb-\udffd\udfff]|\ud83d\udc69\ud83c\udffe\u200d\ud83e\udd1d\u200d\ud83d\udc69\ud83c[\udffb-\udffd\udfff]|\ud83d\udc69\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc68\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83d\udc69\ud83c[\udffb-\udfff]|\ud83d\udc69\ud83c\udfff\u200d\ud83e\udd1d\u200d\ud83d\udc68\ud83c[\udffb-\udffe]|\ud83d\udc69\ud83c\udfff\u200d\ud83e\udd1d\u200d\ud83d\udc69\ud83c[\udffb-\udffe]|\ud83e\uddd1\ud83c\udffb\u200d\u2764\ufe0f\u200d\ud83e\uddd1\ud83c[\udffc-\udfff]|\ud83e\uddd1\ud83c\udffb\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c[\udffb-\udfff]|\ud83e\uddd1\ud83c\udffc\u200d\u2764\ufe0f\u200d\ud83e\uddd1\ud83c[\udffb\udffd-\udfff]|\ud83e\uddd1\ud83c\udffc\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c[\udffb-\udfff]|\ud83e\uddd1\ud83c\udffd\u200d\u2764\ufe0f\u200d\ud83e\uddd1\ud83c[\udffb\udffc\udffe\udfff]|\ud83e\uddd1\ud83c\udffd\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c[\udffb-\udfff]|\ud83e\uddd1\ud83c\udffe\u200d\u2764\ufe0f\u200d\ud83e\uddd1\ud83c[\udffb-\udffd\udfff]|\ud83e\uddd1\ud83c\udffe\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c[\udffb-\udfff]|\ud83e\uddd1\ud83c\udfff\u200d\u2764\ufe0f\u200d\ud83e\uddd1\ud83c[\udffb-\udffe]|\ud83e\uddd1\ud83c\udfff\u200d\ud83e\udd1d\u200d\ud83e\uddd1\ud83c[\udffb-\udfff]|\ud83d\udc68\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d\udc68|\ud83d\udc69\u200d\u2764\ufe0f\u200d\ud83d\udc8b\u200d\ud83d[\udc68\udc69]|\ud83e\udef1\ud83c\udffb\u200d\ud83e\udef2\ud83c[\udffc-\udfff]|\ud83e\udef1\ud83c\udffc\u200d\ud83e\udef2\ud83c[\udffb\udffd-\udfff]|\ud83e\udef1\ud83c\udffd\u200d\ud83e\udef2\ud83c[\udffb\udffc\udffe\udfff]|\ud83e\udef1\ud83c\udffe\u200d\ud83e\udef2\ud83c[\udffb-\udffd\udfff]|\ud83e\udef1\ud83c\udfff\u200d\ud83e\udef2\ud83c[\udffb-\udffe]|\ud83d\udc68\u200d\u2764\ufe0f\u200d\ud83d\udc68|\ud83d\udc69\u200d\u2764\ufe0f\u200d\ud83d[\udc68\udc69]|\ud83e\uddd1\u200d\ud83e\udd1d\u200d\ud83e\uddd1|\ud83d\udc6b\ud83c[\udffb-\udfff]|\ud83d\udc6c\ud83c[\udffb-\udfff]|\ud83d\udc6d\ud83c[\udffb-\udfff]|\ud83d\udc8f\ud83c[\udffb-\udfff]|\ud83d\udc91\ud83c[\udffb-\udfff]|\ud83e\udd1d\ud83c[\udffb-\udfff]|\ud83d[\udc6b-\udc6d\udc8f\udc91]|\ud83e\udd1d)|(?:\ud83d[\udc68\udc69]|\ud83e\uddd1)(?:\ud83c[\udffb-\udfff])?\u200d(?:\u2695\ufe0f|\u2696\ufe0f|\u2708\ufe0f|\ud83c[\udf3e\udf73\udf7c\udf84\udf93\udfa4\udfa8\udfeb\udfed]|\ud83d[\udcbb\udcbc\udd27\udd2c\ude80\ude92]|\ud83e[\uddaf-\uddb3\uddbc\uddbd])(?:\u200d\u27a1\ufe0f)?|(?:\ud83c[\udfcb\udfcc]|\ud83d[\udd74\udd75]|\u26f9)((?:\ud83c[\udffb-\udfff]|\ufe0f)\u200d[\u2640\u2642]\ufe0f(?:\u200d\u27a1\ufe0f)?)|(?:\ud83c[\udfc3\udfc4\udfca]|\ud83d[\udc6e\udc70\udc71\udc73\udc77\udc81\udc82\udc86\udc87\ude45-\ude47\ude4b\ude4d\ude4e\udea3\udeb4-\udeb6]|\ud83e[\udd26\udd35\udd37-\udd39\udd3d\udd3e\uddb8\uddb9\uddcd-\uddcf\uddd4\uddd6-\udddd])(?:\ud83c[\udffb-\udfff])?\u200d[\u2640\u2642]\ufe0f(?:\u200d\u27a1\ufe0f)?|(?:\ud83d\udc68\u200d\ud83d\udc68\u200d\ud83d\udc66\u200d\ud83d\udc66|\ud83d\udc68\u200d\ud83d\udc68\u200d\ud83d\udc67\u200d\ud83d[\udc66\udc67]|\ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d\udc66\u200d\ud83d\udc66|\ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d[\udc66\udc67]|\ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc66\u200d\ud83d\udc66|\ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d[\udc66\udc67]|\ud83e\uddd1\u200d\ud83e\uddd1\u200d\ud83e\uddd2\u200d\ud83e\uddd2|\ud83d\udc68\u200d\ud83d\udc66\u200d\ud83d\udc66|\ud83d\udc68\u200d\ud83d\udc67\u200d\ud83d[\udc66\udc67]|\ud83d\udc68\u200d\ud83d\udc68\u200d\ud83d[\udc66\udc67]|\ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d[\udc66\udc67]|\ud83d\udc69\u200d\ud83d\udc66\u200d\ud83d\udc66|\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d[\udc66\udc67]|\ud83d\udc69\u200d\ud83d\udc69\u200d\ud83d[\udc66\udc67]|\ud83e\uddd1\u200d\ud83e\uddd1\u200d\ud83e\uddd2|\ud83e\uddd1\u200d\ud83e\uddd2\u200d\ud83e\uddd2|\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f|\ud83c\udff3\ufe0f\u200d\ud83c\udf08|\ud83d\ude36\u200d\ud83c\udf2b\ufe0f|\u26d3\ufe0f\u200d\ud83d\udca5|\u2764\ufe0f\u200d\ud83d\udd25|\u2764\ufe0f\u200d\ud83e\ude79|\ud83c\udf44\u200d\ud83d\udfeb|\ud83c\udf4b\u200d\ud83d\udfe9|\ud83c\udff4\u200d\u2620\ufe0f|\ud83d\udc15\u200d\ud83e\uddba|\ud83d\udc26\u200d\ud83d\udd25|\ud83d\udc3b\u200d\u2744\ufe0f|\ud83d\udc41\u200d\ud83d\udde8|\ud83d\udc68\u200d\ud83d[\udc66\udc67]|\ud83d\udc69\u200d\ud83d[\udc66\udc67]|\ud83d\udc6f\u200d\u2640\ufe0f|\ud83d\udc6f\u200d\u2642\ufe0f|\ud83d\ude2e\u200d\ud83d\udca8|\ud83d\ude35\u200d\ud83d\udcab|\ud83d\ude42\u200d\u2194\ufe0f|\ud83d\ude42\u200d\u2195\ufe0f|\ud83e\udd3c\u200d\u2640\ufe0f|\ud83e\udd3c\u200d\u2642\ufe0f|\ud83e\uddd1\u200d\ud83e\uddd2|\ud83e\uddde\u200d\u2640\ufe0f|\ud83e\uddde\u200d\u2642\ufe0f|\ud83e\udddf\u200d\u2640\ufe0f|\ud83e\udddf\u200d\u2642\ufe0f|\ud83d\udc08\u200d\u2b1b|\ud83d\udc26\u200d\u2b1b)|[#*0-9]\ufe0f?\u20e3|(?:[©®\u2122\u265f]\ufe0f)|(?:\ud83c[\udc04\udd70\udd71\udd7e\udd7f\ude02\ude1a\ude2f\ude37\udf21\udf24-\udf2c\udf36\udf7d\udf96\udf97\udf99-\udf9b\udf9e\udf9f\udfcd\udfce\udfd4-\udfdf\udff3\udff5\udff7]|\ud83d[\udc3f\udc41\udcfd\udd49\udd4a\udd6f\udd70\udd73\udd76-\udd79\udd87\udd8a-\udd8d\udda5\udda8\uddb1\uddb2\uddbc\uddc2-\uddc4\uddd1-\uddd3\udddc-\uddde\udde1\udde3\udde8\uddef\uddf3\uddfa\udecb\udecd-\udecf\udee0-\udee5\udee9\udef0\udef3]|[\u203c\u2049\u2139\u2194-\u2199\u21a9\u21aa\u231a\u231b\u2328\u23cf\u23ed-\u23ef\u23f1\u23f2\u23f8-\u23fa\u24c2\u25aa\u25ab\u25b6\u25c0\u25fb-\u25fe\u2600-\u2604\u260e\u2611\u2614\u2615\u2618\u2620\u2622\u2623\u2626\u262a\u262e\u262f\u2638-\u263a\u2640\u2642\u2648-\u2653\u2660\u2663\u2665\u2666\u2668\u267b\u267f\u2692-\u2697\u2699\u269b\u269c\u26a0\u26a1\u26a7\u26aa\u26ab\u26b0\u26b1\u26bd\u26be\u26c4\u26c5\u26c8\u26cf\u26d1\u26d3\u26d4\u26e9\u26ea\u26f0-\u26f5\u26f8\u26fa\u26fd\u2702\u2708\u2709\u270f\u2712\u2714\u2716\u271d\u2721\u2733\u2734\u2744\u2747\u2757\u2763\u2764\u27a1\u2934\u2935\u2b05-\u2b07\u2b1b\u2b1c\u2b50\u2b55\u3030\u303d\u3297\u3299])(?:\ufe0f|(?!\ufe0e))|(?:(?:\ud83c[\udfcb\udfcc]|\ud83d[\udd74\udd75\udd90]|\ud83e\udef0|[\u261d\u26f7\u26f9\u270c\u270d])(?:\ufe0f|(?!\ufe0e))|(?:\ud83c\udfc3|\ud83d\udeb6|\ud83e\uddce)(?:\ud83c[\udffb-\udfff])?(?:\u200d\u27a1\ufe0f)?|(?:\ud83c[\udf85\udfc2\udfc4\udfc7\udfca]|\ud83d[\udc42\udc43\udc46-\udc50\udc66-\udc69\udc6e\udc70-\udc78\udc7c\udc81-\udc83\udc85-\udc87\udcaa\udd7a\udd95\udd96\ude45-\ude47\ude4b-\ude4f\udea3\udeb4\udeb5\udec0\udecc]|\ud83e[\udd0c\udd0f\udd18-\udd1c\udd1e\udd1f\udd26\udd30-\udd39\udd3d\udd3e\udd77\uddb5\uddb6\uddb8\uddb9\uddbb\uddcd\uddcf\uddd1-\udddd\udec3-\udec5\udef1-\udef8]|[\u270a\u270b]))(?:\ud83c[\udffb-\udfff])?|(?:\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f|\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc73\udb40\udc63\udb40\udc74\udb40\udc7f|\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc77\udb40\udc6c\udb40\udc73\udb40\udc7f|\ud83c\udde6\ud83c[\udde8-\uddec\uddee\uddf1\uddf2\uddf4\uddf6-\uddfa\uddfc\uddfd\uddff]|\ud83c\udde7\ud83c[\udde6\udde7\udde9-\uddef\uddf1-\uddf4\uddf6-\uddf9\uddfb\uddfc\uddfe\uddff]|\ud83c\udde8\ud83c[\udde6\udde8\udde9\uddeb-\uddee\uddf0-\uddf7\uddfa-\uddff]|\ud83c\udde9\ud83c[\uddea\uddec\uddef\uddf0\uddf2\uddf4\uddff]|\ud83c\uddea\ud83c[\udde6\udde8\uddea\uddec\udded\uddf7-\uddfa]|\ud83c\uddeb\ud83c[\uddee-\uddf0\uddf2\uddf4\uddf7]|\ud83c\uddec\ud83c[\udde6\udde7\udde9-\uddee\uddf1-\uddf3\uddf5-\uddfa\uddfc\uddfe]|\ud83c\udded\ud83c[\uddf0\uddf2\uddf3\uddf7\uddf9\uddfa]|\ud83c\uddee\ud83c[\udde8-\uddea\uddf1-\uddf4\uddf6-\uddf9]|\ud83c\uddef\ud83c[\uddea\uddf2\uddf4\uddf5]|\ud83c\uddf0\ud83c[\uddea\uddec-\uddee\uddf2\uddf3\uddf5\uddf7\uddfc\uddfe\uddff]|\ud83c\uddf1\ud83c[\udde6-\udde8\uddee\uddf0\uddf7-\uddfb\uddfe]|\ud83c\uddf2\ud83c[\udde6\udde8-\udded\uddf0-\uddff]|\ud83c\uddf3\ud83c[\udde6\udde8\uddea-\uddec\uddee\uddf1\uddf4\uddf5\uddf7\uddfa\uddff]|\ud83c\uddf4\ud83c\uddf2|\ud83c\uddf5\ud83c[\udde6\uddea-\udded\uddf0-\uddf3\uddf7-\uddf9\uddfc\uddfe]|\ud83c\uddf6\ud83c\udde6|\ud83c\uddf7\ud83c[\uddea\uddf4\uddf8\uddfa\uddfc]|\ud83c\uddf8\ud83c[\udde6-\uddea\uddec-\uddf4\uddf7-\uddf9\uddfb\uddfd-\uddff]|\ud83c\uddf9\ud83c[\udde6\udde8\udde9\uddeb-\udded\uddef-\uddf4\uddf7\uddf9\uddfb\uddfc\uddff]|\ud83c\uddfa\ud83c[\udde6\uddec\uddf2\uddf3\uddf8\uddfe\uddff]|\ud83c\uddfb\ud83c[\udde6\udde8\uddea\uddec\uddee\uddf3\uddfa]|\ud83c\uddfc\ud83c[\uddeb\uddf8]|\ud83c\uddfd\ud83c\uddf0|\ud83c\uddfe\ud83c[\uddea\uddf9]|\ud83c\uddff\ud83c[\udde6\uddf2\uddfc]|\ud83c[\udccf\udd8e\udd91-\udd9a\udde6-\uddff\ude01\ude32-\ude36\ude38-\ude3a\ude50\ude51\udf00-\udf20\udf2d-\udf35\udf37-\udf7c\udf7e-\udf84\udf86-\udf93\udfa0-\udfc1\udfc5\udfc6\udfc8\udfc9\udfcf-\udfd3\udfe0-\udff0\udff4\udff8-\udfff]|\ud83d[\udc00-\udc3e\udc40\udc44\udc45\udc51-\udc65\udc6a\udc6f\udc79-\udc7b\udc7d-\udc80\udc84\udc88-\udc8e\udc90\udc92-\udca9\udcab-\udcfc\udcff-\udd3d\udd4b-\udd4e\udd50-\udd67\udda4\uddfb-\ude44\ude48-\ude4a\ude80-\udea2\udea4-\udeb3\udeb7-\udebf\udec1-\udec5\uded0-\uded2\uded5-\uded7\udedc-\udedf\udeeb\udeec\udef4-\udefc\udfe0-\udfeb\udff0]|\ud83e[\udd0d\udd0e\udd10-\udd17\udd20-\udd25\udd27-\udd2f\udd3a\udd3c\udd3f-\udd45\udd47-\udd76\udd78-\uddb4\uddb7\uddba\uddbc-\uddcc\uddd0\uddde-\uddff\ude70-\ude7c\ude80-\ude89\ude8f-\udec2\udec6\udece-\udedc\udedf-\udee9]|[\u23e9-\u23ec\u23f0\u23f3\u267e\u26ce\u2705\u2728\u274c\u274e\u2753-\u2755\u2795-\u2797\u27b0\u27bf\ue50a])|\ufe0f/g; export const emojiRegex = new RegExp(`(${twemojiRegex.source})`); export const emojiRegexAtStartToEnd = new RegExp(`^(${twemojiRegex.source})$`); diff --git a/packages/backend/src/misc/errors/AbortedError.ts b/packages/backend/src/misc/errors/AbortedError.ts new file mode 100644 index 0000000000..99a0581fc4 --- /dev/null +++ b/packages/backend/src/misc/errors/AbortedError.ts @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { renderInlineError } from '@/misc/render-inline-error.js'; + +/** + * Throw when an operation is unexpectedly aborted. + */ +export class AbortedError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + + constructor(signal: AbortSignal, message?: string, options?: Omit) { + super( + `Operation aborted: ${message ?? renderInlineError(signal.reason)}`, + { + ...(options ?? {}), + cause: signal.reason, + }, + ); + } +} diff --git a/packages/backend/src/misc/errors/ConflictError.ts b/packages/backend/src/misc/errors/ConflictError.ts new file mode 100644 index 0000000000..ecd9919589 --- /dev/null +++ b/packages/backend/src/misc/errors/ConflictError.ts @@ -0,0 +1,9 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class ConflictError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; +} diff --git a/packages/backend/src/misc/errors/DisposeError.ts b/packages/backend/src/misc/errors/DisposeError.ts new file mode 100644 index 0000000000..deba6450fd --- /dev/null +++ b/packages/backend/src/misc/errors/DisposeError.ts @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * Common base class for DisposedError and DisposingError - please use only for catch() blocks. + */ +export abstract class DisposeError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + + public readonly source: string | undefined; + + protected constructor(opts?: { source?: string, message?: string }) { + super(opts?.message); + this.source = opts?.source; + } +} + +/** + * Thrown when an attempt is made to use an object that has been disposed. + */ +export class DisposedError extends DisposeError { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + + constructor(opts?: { source?: string, message?: string }) { + super({ + source: opts?.source, + message: opts?.message ?? `${opts?.source ?? 'Object'} has been disposed`, + }); + } +} + +/** + * Thrown when an object is use begins disposing. + */ +export class DisposingError extends DisposeError { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + + constructor(opts?: { source?: string, message?: string }) { + super({ + source: opts?.source, + message: opts?.message ?? `${opts?.source ?? 'Object'} is being disposed`, + }); + } +} diff --git a/packages/backend/src/misc/errors/FetchFailedError.ts b/packages/backend/src/misc/errors/FetchFailedError.ts new file mode 100644 index 0000000000..48813e36a4 --- /dev/null +++ b/packages/backend/src/misc/errors/FetchFailedError.ts @@ -0,0 +1,33 @@ +import { QuantumCacheError } from '@/misc/errors/QuantumCacheError.js'; + +/** + * Thrown when a fetch failed for any reason. + */ +export class FetchFailedError extends QuantumCacheError { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + + /** + * Name of the key(s) that could not be fetched. + * Will be an array if bulkFetcher() failed, and a string if regular fetch() failed. + */ + public readonly keyNames: string | readonly string[]; + + constructor( + cacheName: string, + keyNames: string | readonly string[], + message?: string, + options?: ErrorOptions, + ) { + const actualMessage = typeof (keyNames) === 'string' + ? message + ? `Fetch failed for key "${keyNames}": ${message}` + : `Fetch failed for key "${keyNames}".` + : message + ? `Fetch failed for ${keyNames.length} keys: ${message}` + : `Fetch failed for ${keyNames.length} keys.`; + super(cacheName, actualMessage, options); + + this.keyNames = keyNames; + } +} diff --git a/packages/backend/src/misc/errors/KeyNotFoundError.ts b/packages/backend/src/misc/errors/KeyNotFoundError.ts new file mode 100644 index 0000000000..dec84c1dfb --- /dev/null +++ b/packages/backend/src/misc/errors/KeyNotFoundError.ts @@ -0,0 +1,27 @@ +import { FetchFailedError } from '@/misc/errors/FetchFailedError.js'; +import { isRetryableSymbol } from '@/misc/is-retryable-error.js'; + +/** + * Thrown when a fetch failed because no value was found for the requested key(s). + */ +export class KeyNotFoundError extends FetchFailedError { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + + /** + * Missing keys are considered non-retryable, as they won't suddenly appear unless something external creates them. + */ + readonly [isRetryableSymbol] = false; + + constructor( + cacheName: string, + keyNames: string | readonly string[], + message?: string, + options?: ErrorOptions, + ) { + const actualMessage = message + ? `Fetcher did not return a value: ${message}` + : 'Fetcher did not return a value.'; + super(cacheName, keyNames, actualMessage, options); + } +} diff --git a/packages/backend/src/misc/errors/QuantumCacheError.ts b/packages/backend/src/misc/errors/QuantumCacheError.ts new file mode 100644 index 0000000000..fc329697fa --- /dev/null +++ b/packages/backend/src/misc/errors/QuantumCacheError.ts @@ -0,0 +1,25 @@ +/** + * Base class for all Quantum Cache errors. + */ +export class QuantumCacheError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + + /** + * Name of the cache that produced this error. + */ + public readonly cacheName: string; + + constructor( + cacheName: string, + message?: string, + options?: ErrorOptions, + ) { + const actualMessage = message + ? `Error in cache ${cacheName}: ${message}` + : `Error in cache ${cacheName}.`; + super(actualMessage, options); + + this.cacheName = cacheName; + } +} diff --git a/packages/backend/src/misc/fastify-reply-error.ts b/packages/backend/src/misc/fastify-reply-error.ts index 03109e8b96..885d33fad3 100644 --- a/packages/backend/src/misc/fastify-reply-error.ts +++ b/packages/backend/src/misc/fastify-reply-error.ts @@ -5,6 +5,9 @@ // https://www.fastify.io/docs/latest/Reference/Reply/#async-await-and-promises export class FastifyReplyError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + public message: string; public statusCode: number; diff --git a/packages/backend/src/misc/generate-invite-code.ts b/packages/backend/src/misc/generate-invite-code.ts index 006920cf0e..377215fd30 100644 --- a/packages/backend/src/misc/generate-invite-code.ts +++ b/packages/backend/src/misc/generate-invite-code.ts @@ -7,13 +7,13 @@ import { secureRndstr } from './secure-rndstr.js'; const CHARS = '23456789ABCDEFGHJKLMNPQRSTUVWXYZ'; // [0-9A-Z] w/o [01IO] (32 patterns) -export function generateInviteCode(): string { +export function generateInviteCode(now: number): string { const code = secureRndstr(8, { chars: CHARS, }); - const uniqueId = []; - let n = Math.floor(Date.now() / 1000 / 60); + const uniqueId: string[] = []; + let n = Math.floor(now / 1000 / 60); while (true) { uniqueId.push(CHARS[n % CHARS.length]); const t = Math.floor(n / CHARS.length); diff --git a/packages/backend/src/misc/get-note-summary.ts b/packages/backend/src/misc/get-note-summary.ts index be183b4979..c6a5493531 100644 --- a/packages/backend/src/misc/get-note-summary.ts +++ b/packages/backend/src/misc/get-note-summary.ts @@ -6,11 +6,49 @@ import { appendContentWarning } from './append-content-warning.js'; import type { Packed } from './json-schema.js'; +// export type PackedNoteForSummary = Omit>, 'user'> & { +// user: Omit['user']>, 'instance'> & { +// instance?: Partial['user']['instance']> | null; +// }; +// }; + +// Workaround for weird typescript but +// type Pivot = { +// [KNote in keyof N]?: KNote extends 'user' +// ? { +// [KUser in keyof U]?: KUser extends 'instance' +// ? { +// [KInst in keyof I]?: I[KInst] | undefined; +// } +// : U[KUser] +// } +// : N[KNote] +// }; +// type Pivot = Split & { +// user: Split & { +// instance: I | undefined; +// }; +// }; +// +// type Split> = { +// [K in (keyof T) & R]: T[K]; +// } & { +// [K in (keyof T) & O]?: T[K] | undefined; +// }; +// +// export type PackedNoteForSummary = Pivot, Packed<'Note'>['user'], NonNullable['user']['instance']>>; +export type PackedNoteForSummary = DeepPartial>; + +// Do we really not have a type for this yet?? +type DeepPartial = { + [K in keyof T]?: T[K] extends object ? DeepPartial : T[K]; +}; + /** * 投稿を表す文字列を取得します。 * @param {*} note (packされた)投稿 */ -export const getNoteSummary = (note: Packed<'Note'>): string => { +export const getNoteSummary = (note: PackedNoteForSummary): string => { if (note.deletedAt) { return '(❌⛔)'; } @@ -26,13 +64,13 @@ export const getNoteSummary = (note: Packed<'Note'>): string => { if (note.mandatoryCW) { cw = appendContentWarning(cw, `Note is flagged: "${note.mandatoryCW}"`); } - if (note.user.mandatoryCW) { + if (note.user?.mandatoryCW) { const username = note.user.host ? `@${note.user.username}@${note.user.host}` : `@${note.user.username}`; cw = appendContentWarning(cw, `${username} is flagged: "${note.user.mandatoryCW}"`); } - if (note.user.instance?.mandatoryCW) { + if (note.user?.instance?.mandatoryCW) { cw = appendContentWarning(cw, `${note.user.host} is flagged: "${note.user.instance.mandatoryCW}"`); } diff --git a/packages/backend/src/misc/i18n.ts b/packages/backend/src/misc/i18n.ts index 6cbbdef74c..e32c1a028b 100644 --- a/packages/backend/src/misc/i18n.ts +++ b/packages/backend/src/misc/i18n.ts @@ -3,10 +3,18 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { LoggerService } from '@/core/LoggerService.js'; +import type Logger from '@/logger.js'; + export class I18n> { + private readonly logger: Logger; public locale: T; - constructor(locale: T) { + constructor( + loggerService: LoggerService, + locale: T, + ) { + this.logger = loggerService.getLogger('i18n'); this.locale = locale; //#region BIND @@ -26,8 +34,8 @@ export class I18n> { } } return str; - } catch (e) { - console.warn(`missing localization '${key}'`); + } catch { + this.logger.warn(`missing localization '${key}'`); return key; } } diff --git a/packages/backend/src/misc/identifiable-error.ts b/packages/backend/src/misc/identifiable-error.ts index 56e13f2622..9497791ea1 100644 --- a/packages/backend/src/misc/identifiable-error.ts +++ b/packages/backend/src/misc/identifiable-error.ts @@ -7,6 +7,9 @@ * ID付きエラー */ export class IdentifiableError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + public message: string; public id: string; diff --git a/packages/backend/src/misc/is-error.ts b/packages/backend/src/misc/is-error.ts new file mode 100644 index 0000000000..1e1f00e48c --- /dev/null +++ b/packages/backend/src/misc/is-error.ts @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { types } from 'node:util'; + +const hasDOMException = Reflect.has(globalThis, 'DOMException'); +const hasErrorIsError = Reflect.has(globalThis.Error, 'isError'); + +/** + * Polyfill for https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error/isError + */ +export const isError: IsErrorFunc = hasErrorIsError ? Error.isError : isErrorPolyfill; + +/** + * Returns true if error is an instance of Error, false otherwise. + * More robust than instanceof. + */ +export type IsErrorFunc = (error: unknown) => error is Error; + +export function isErrorPolyfill(error: unknown): error is Error { + // These are the fastest checks, so run them first + if (isErrorByInstance(error)) { + return true; + } + + // Errors must be a non-null object + if (typeof(error) !== 'object' || error == null) { + return false; + } + + // jest, and maybe a few other edge cases + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Symbol/species + if ('constructor' in error && isErrorByInstance(error.constructor[Symbol.species])) { + return true; + } + + // If it looks like a duck and quacks like a duck... + if ('name' in error && typeof(error.name) === 'string') { + if ('message' in error && typeof(error.message) === 'string') { + if (!('stack' in error) || typeof(error.stack) === 'string' || typeof(error.stack) === 'undefined') { + return true; + } + } + } + + // Guess it's not :( + return false; +} + +function isErrorByInstance(error: unknown): boolean { + // It must be a non-null object + if (typeof(error) !== 'object' || error == null) { + return false; + } + + // An actual instance, nice + if (error instanceof Error) { + return true; + } + + // DOMException is an Error, just without the prototype chain + if (hasDOMException && error instanceof DOMException) { + return true; + } + + // Realm fuckery + if (types.isNativeError(error)) { + return true; + } + + return false; +} + diff --git a/packages/backend/src/misc/is-retryable-error.ts b/packages/backend/src/misc/is-retryable-error.ts index 63b561b280..b83aefa81f 100644 --- a/packages/backend/src/misc/is-retryable-error.ts +++ b/packages/backend/src/misc/is-retryable-error.ts @@ -3,19 +3,25 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +// TODO replace all direct imports w/ the symbol import { AbortError, FetchError } from 'node-fetch'; import { UnrecoverableError } from 'bullmq'; import { StatusError } from '@/misc/status-error.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { CaptchaError, captchaErrorCodes } from '@/core/CaptchaService.js'; +import { CaptchaError, captchaErrorCodes } from '@/misc/captcha-error.js'; import { FastifyReplyError } from '@/misc/fastify-reply-error.js'; -import { ConflictError } from '@/server/SkRateLimiterService.js'; +import { ConflictError } from '@/misc/errors/ConflictError.js'; /** * Returns false if the provided value represents a "permanent" error that cannot be retried. * Returns true if the error is retryable, unknown (as all errors are retryable by default), or not an error object. + * If the error cannot be readily identified as retryable, then recurses to the inner exception ("cause" property). */ export function isRetryableError(e: unknown): boolean { + if (hasRetryableSymbol(e)) { + return e[isRetryableSymbol]; + } + if (e instanceof AggregateError) return e.errors.every(inner => isRetryableError(inner)); if (e instanceof StatusError) return e.isRetryable; if (e instanceof IdentifiableError) return e.isRetryable; @@ -29,8 +35,29 @@ export function isRetryableError(e: unknown): boolean { if (e instanceof ConflictError) return true; if (e instanceof UnrecoverableError) return false; if (e instanceof AbortError) return true; - if (e instanceof FetchError) return true; + if (e instanceof FetchError) return true; // TODO check status code? if (e instanceof SyntaxError) return false; - if (e instanceof Error) return e.name === 'AbortError'; + if (e instanceof Error) { + if (e.name === 'AbortError') return true; + if (e.cause != null) return isRetryableError(e.cause); + } + + // TODO aggregate errors (any permanent makes the whole thing permanent) + // TODO "got" errors + return true; } + +/** + * Error classes may define a gettable property with this key to directly specify retryability. + * If the property resolves to a boolean, then that value will be used. + * Returning any other value will fall back on the usual logic. + */ +export const isRetryableSymbol = Symbol('isRetryable'); + +function hasRetryableSymbol(obj: unknown): obj is { [isRetryableSymbol]: boolean } { + return obj != null + && typeof(obj) === 'object' + && isRetryableSymbol in obj + && typeof(obj[isRetryableSymbol]) === 'boolean'; +} diff --git a/packages/backend/src/misc/kvp-array.ts b/packages/backend/src/misc/kvp-array.ts new file mode 100644 index 0000000000..8ed8f230c1 --- /dev/null +++ b/packages/backend/src/misc/kvp-array.ts @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * Key-Value Pair Array - stores a collection of Key/Value pairs with helper methods to access ordered keys/values. + * Keys and Values can be of any type, and Keys default to type "string" if unspecified. + */ +export type KVPArray = KVPs & { + /** + * Lazy-loaded array of all keys in the array, matching the order of the pairs. + */ + readonly keys: readonly K[], + + /** + * Lazy-loaded array of all values in the array, matching the order of the pairs. + */ + readonly values: readonly T[], +}; + +type KVPs = Omit[], 'keys' | 'values' | 'entries'>; +type KVP = readonly [key: K, value: T]; + +/** + * Wraps an array of Key/Value pairs into a KVPArray. + */ +export function makeKVPArray(pairs: KVPs): KVPArray { + let keys: K[] | null = null; + let values: T[] | null = null; + + Object.defineProperties(pairs, { + keys: { + get() { + return keys ??= pairs.map(pair => pair[0]); + }, + enumerable: false, + }, + values: { + get() { + return values ??= pairs.map(pair => pair[1]); + }, + enumerable: false, + }, + }); + + return pairs as KVPArray; +} diff --git a/packages/backend/src/misc/patch-date.ts b/packages/backend/src/misc/patch-date.ts new file mode 100644 index 0000000000..f7d21a3915 --- /dev/null +++ b/packages/backend/src/misc/patch-date.ts @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * Replaces specific properties of a Date object while leaving all others unchanged. + * Returns a new date, does not mutate the original. + * @param date Reference date to modify. + * @param patch Values to replace. + */ +export function patchDate(date: Date, patch: DatePatch): Date { + return new Date( + patch.year ?? date.getFullYear(), + patch.month ?? date.getMonth(), // month number -1 + patch.day ?? date.getDate(), // getDate, not getDay!! + patch.hours ?? date.getHours(), + patch.minutes ?? date.getMinutes(), + patch.seconds ?? date.getSeconds(), + patch.milliseconds ?? date.getMilliseconds(), + ); +} + +/** + * Increments specific properties of a Date object while leaving all others unchanged. + * Returns a new date, does not mutate the original. + * @param date Reference date to modify. + * @param patch Values to add. + */ +export function addPatch(date: Date, patch: DatePatch): Date { + return new Date( + (patch.year ?? 0) + date.getFullYear(), + (patch.month ?? 0) + date.getMonth(), // month number -1 + (patch.day ?? 0) + date.getDate(), // getDate, not getDay!! + (patch.hours ?? 0) + date.getHours(), + (patch.minutes ?? 0) + date.getMinutes(), + (patch.seconds ?? 0) + date.getSeconds(), + (patch.milliseconds ?? 0) + date.getMilliseconds(), + ); +} + +/** + * Extracts a DatePatch containing the current values from a given Date. + */ +export function toPatch(date: Date): DatePatch { + return { + year: date.getFullYear(), + month: date.getMonth(), // month number -1 + day: date.getDate(), // getDate, not getDay!! + hours: date.getHours(), + minutes: date.getMinutes(), + seconds: date.getSeconds(), + milliseconds: date.getMilliseconds(), + }; +} + +/** + * Produces a Date from the values of a given DatePatch. + */ +export function fromPatch(patch: DatePatch): Date { + return new Date( + patch.year ?? 0, + patch.month ?? 0, + patch.day ?? 0, + patch.hours ?? 0, + patch.minutes ?? 0, + patch.seconds ?? 0, + patch.milliseconds ?? 0, + ); +} + +export interface DatePatch { + year?: number, + month?: number, + day?: number, + hours?: number, + minutes?: number, + seconds?: number, + milliseconds?: number, +} diff --git a/packages/backend/src/misc/prelude/array.ts b/packages/backend/src/misc/prelude/array.ts index f741a0c913..95ec9c65ed 100644 --- a/packages/backend/src/misc/prelude/array.ts +++ b/packages/backend/src/misc/prelude/array.ts @@ -80,7 +80,7 @@ export function lessThan(xs: number[], ys: number[]): boolean { * Returns the longest prefix of elements that satisfy the predicate */ export function takeWhile(f: Predicate, xs: T[]): T[] { - const ys = []; + const ys: T[] = []; for (const x of xs) { if (f(x)) { ys.push(x); diff --git a/packages/backend/src/misc/promise-tracker.ts b/packages/backend/src/misc/promise-tracker.ts index 76b4dd810c..d4411df17c 100644 --- a/packages/backend/src/misc/promise-tracker.ts +++ b/packages/backend/src/misc/promise-tracker.ts @@ -3,23 +3,28 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { coreLogger } from '@/boot/coreLogger.js'; + +const backgroundLogger = coreLogger.createSubLogger('background'); const promiseRefs: Set>> = new Set(); -export function trackTask(task: () => Promise): void { - trackPromise(task()); +export function trackTask(task: () => Promise): Promise { + const promise = task(); + return trackPromise(promise); } /** * This tracks promises that other modules decided not to wait for, * and makes sure they are all settled before fully closing down the server. + * Returns the promise for chaining. */ -export function trackPromise(promise: Promise) { - if (process.env.NODE_ENV !== 'test') { - return; - } +export function trackPromise(promise: Promise): Promise { const ref = new WeakRef(promise); promiseRefs.add(ref); - promise.finally(() => promiseRefs.delete(ref)); + promise + .catch(err => backgroundLogger.error('Unhandled error in tracked background task:', { err })) + .finally(() => promiseRefs.delete(ref)); + return promise; } export async function allSettled(): Promise { diff --git a/packages/backend/src/misc/promise-try.ts b/packages/backend/src/misc/promise-try.ts new file mode 100644 index 0000000000..2b1594b2ee --- /dev/null +++ b/packages/backend/src/misc/promise-try.ts @@ -0,0 +1,34 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { types } from 'node:util'; + +const hasPromiseTry = Reflect.has(globalThis.Promise, 'try'); + +/** + * Polyfill for https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/try + */ +export const promiseTry: PromiseTryFunc = hasPromiseTry ? Promise.try : promiseTryPolyfill; + +/** + * Takes a callback of any kind (returns or throws, synchronously or asynchronously) and wraps its result in a Promise. + */ +export type PromiseTryFunc = (callbackFn: (...args: U) => T | PromiseLike, ...args: U) => Promise>; + +export function promiseTryPolyfill(callbackFn: (...args: U) => T | PromiseLike, ...args: U): Promise> { + try { + const result = callbackFn(...args); + if (types.isPromise(result)) { + // async return or throw + return result as Promise>; + } + // sync return + return Promise.resolve(result); + } catch (err) { + // sync throw + return Promise.reject(err); + } +} + diff --git a/packages/backend/src/misc/promiseUtils.ts b/packages/backend/src/misc/promiseUtils.ts new file mode 100644 index 0000000000..d3e754582c --- /dev/null +++ b/packages/backend/src/misc/promiseUtils.ts @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { throwIfAborted } from '@/misc/throw-if-aborted.js'; +import { AbortedError } from '@/misc/errors/AbortedError.js'; + +/** + * Executes a task or promise, then runs a provided cleanup task. + * The resulting task resolves only when *both* steps are complete. + * One or both of the steps may throw, but the other will always run anyway. + * All errors are captured, aggregated, and re-thrown by the final promise. + * + * @param promiseOrCallback Promise or async callback to execute + * @param cleanup Cleanup callback to execute after execution completes or fails + */ +export async function withCleanup(promiseOrCallback: MaybeCallback>, cleanup: () => MaybePromise): Promise { + // Execute the task first + let executionResult: Result; + try { + const result = typeof(promiseOrCallback) === 'function' + ? await promiseOrCallback() + : await promiseOrCallback; + executionResult = { success: true, result }; + } catch (error) { + executionResult = { success: false, error }; + } + + // Run cleanup next, even if execution failed + let cleanupResult: Result; + try { + const result = await cleanup(); + cleanupResult = { success: true, result }; + } catch (error) { + cleanupResult = { success: false, error }; + } + + if (!executionResult.success) { + if (!cleanupResult.success) { + // Execution and cleanup failed + throw new AggregateError([executionResult.error, cleanupResult.error]); + } else { + // Execution failed, but cleanup succeeded + throw executionResult.error; + } + } + + // Execution succeeded, but cleanup failed + if (!cleanupResult.success) { + throw cleanupResult.error; + } + + // Execution and cleanup succeeded + return executionResult.result; +} + +/** + * Binds an AbortSignal to a Promise. + * The returned promise will resolve or reject with the result of the provided promise, unless the signal is aborted first. + * + * The promise must be provided as an async factory, which will be called to produce the actual task promise. + * This requirement is in place to ensure consistent behavior if the abortSignal is already aborted. + * Otherwise, the input promise may produce an UnhandledPromiseRejection error that crashes the app. + * @param factory Callback to start the promise + * @param abortSignal Signal to terminate the promise + */ // TODO accept a promise directly here +export async function withSignal(factory: () => Promise, abortSignal: AbortSignal): Promise { + // If already aborted, then don't do anything. + throwIfAborted(abortSignal); + + // Create a promise with controls. + const { promise, resolve, reject } = Promise.withResolvers(); + const abort = () => reject(new AbortedError(abortSignal)); + + // Bind the abort signal. + abortSignal.addEventListener('abort', abort); + promise + .finally(() => abortSignal.removeEventListener('abort', abort)) + .catch(() => null); // Make sure it's never an unhandled rejection! + + // Bind the task promise. + const taskPromise = factory(); + taskPromise + .then(result => resolve(result), err => reject(err)) + .catch(() => null); // Make sure it's never an unhandled rejection! + + return promise; +} + +type Result = + { success: true, result: T } | + { success: false, error: unknown }; + +type MaybeCallback = T | (() => T); +type MaybePromise = T | Promise; diff --git a/packages/backend/src/misc/render-full-error.ts b/packages/backend/src/misc/render-full-error.ts index 5f0a09bba9..7433c7ac34 100644 --- a/packages/backend/src/misc/render-full-error.ts +++ b/packages/backend/src/misc/render-full-error.ts @@ -8,7 +8,7 @@ import { AbortError, FetchError } from 'node-fetch'; import { StatusError } from '@/misc/status-error.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; -import { CaptchaError, captchaErrorCodes } from '@/core/CaptchaService.js'; +import { CaptchaError, captchaErrorCodes } from '@/misc/captcha-error.js'; export function renderFullError(e?: unknown): unknown { if (e === undefined) return 'undefined'; diff --git a/packages/backend/src/misc/reset-db.ts b/packages/backend/src/misc/reset-db.ts index 75fb4c3e7b..3c1606df39 100644 --- a/packages/backend/src/misc/reset-db.ts +++ b/packages/backend/src/misc/reset-db.ts @@ -24,6 +24,8 @@ export async function resetDb(db: DataSource) { if (i === 3) { throw e; } else { + // Ignore rule - this is just testing code. + // eslint-disable-next-line no-restricted-globals await new Promise(resolve => setTimeout(resolve, 1000)); continue; } diff --git a/packages/backend/src/misc/status-error.ts b/packages/backend/src/misc/status-error.ts index 4fd3bfcafb..815a7686e3 100644 --- a/packages/backend/src/misc/status-error.ts +++ b/packages/backend/src/misc/status-error.ts @@ -4,6 +4,9 @@ */ export class StatusError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + public statusCode: number; public statusMessage?: string; public isClientError: boolean; diff --git a/packages/backend/src/misc/throw-if-aborted.ts b/packages/backend/src/misc/throw-if-aborted.ts new file mode 100644 index 0000000000..f4d971bf1b --- /dev/null +++ b/packages/backend/src/misc/throw-if-aborted.ts @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { AbortedError } from '@/misc/errors/AbortedError.js'; + +export function throwIfAborted(signal: AbortSignal): void { + if (signal.aborted) { + throw new AbortedError(signal); + } +} + +export function rejectIfAborted(signal: AbortSignal): Promise { + if (signal.aborted) { + return Promise.reject(new AbortedError(signal)); + } else { + return Promise.resolve(); + } +} diff --git a/packages/backend/src/misc/verify-field-link.ts b/packages/backend/src/misc/verify-field-link.ts index 31a356be37..bc79ad6cb6 100644 --- a/packages/backend/src/misc/verify-field-link.ts +++ b/packages/backend/src/misc/verify-field-link.ts @@ -9,7 +9,7 @@ import type { HttpRequestService } from '@/core/HttpRequestService.js'; type Field = { name: string, value: string }; export async function verifyFieldLinks(fields: Field[], profileUrls: string[], httpRequestService: HttpRequestService): Promise { - const verified_links = []; + const verified_links: string[] = []; for (const field_url of fields) { try { // getHtml validates the input URL, so we can safely pass in untrusted values diff --git a/packages/backend/src/models/Emoji.ts b/packages/backend/src/models/Emoji.ts index 9f31455b83..a4cad34ce5 100644 --- a/packages/backend/src/models/Emoji.ts +++ b/packages/backend/src/models/Emoji.ts @@ -84,4 +84,10 @@ export class MiEmoji { array: true, length: 128, default: '{}', }) public roleIdsThatCanBeUsedThisEmojiAsReaction: string[]; + + constructor(data?: Partial) { + if (data) { + Object.assign(this, data); + } + } } diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index c282fa327f..b2b13e1724 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -430,6 +430,22 @@ export type MiPartialRemoteUser = Partial & { uri: string; }; +export function isRemoteUser(user: MiUser): user is MiRemoteUser; +export function isRemoteUser(user: U): user is PartialRemoteUser; +export function isRemoteUser(user: { host: string | null }): user is { host: string } { + return user.host != null; +} + +export function isLocalUser(user: MiUser): user is MiLocalUser; +export function isLocalUser(user: U): user is PartialLocalUser; +export function isLocalUser(user: { host: string | null }): user is { host: null } { + return user.host == null; +} + +type PartialUser = Partial & { host: string | null, uri?: string | null }; +type PartialLocalUser = U & { host: null, uri?: null }; +type PartialRemoteUser = U & { host: string, uri?: string }; + export const localUsernameSchema = { type: 'string', pattern: /^\w{1,20}$/.toString().slice(1, -1) } as const; export const passwordSchema = { type: 'string', minLength: 1 } as const; export const nameSchema = { type: 'string', minLength: 1, maxLength: 50 } as const; diff --git a/packages/backend/src/models/json-schema/user-list.ts b/packages/backend/src/models/json-schema/user-list.ts index dc9af25602..75dcc991eb 100644 --- a/packages/backend/src/models/json-schema/user-list.ts +++ b/packages/backend/src/models/json-schema/user-list.ts @@ -17,13 +17,18 @@ export const packedUserListSchema = { optional: false, nullable: false, format: 'date-time', }, + createdBy: { + type: 'string', + optional: false, nullable: false, + format: 'id', + }, name: { type: 'string', optional: false, nullable: false, }, userIds: { type: 'array', - nullable: false, optional: true, + nullable: false, optional: false, items: { type: 'string', nullable: false, optional: false, @@ -35,5 +40,15 @@ export const packedUserListSchema = { nullable: false, optional: false, }, + isLiked: { + type: 'boolean', + nullable: false, + optional: true, + }, + likedCount: { + type: 'number', + nullable: false, + optional: true, + }, }, } as const; diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index 45caec54ce..d08adb70b4 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -9,7 +9,9 @@ import { DataSource, Logger, type QueryRunner } from 'typeorm'; import * as highlight from 'cli-highlight'; import { entities as charts } from '@/core/chart/entities.js'; import { Config } from '@/config.js'; -import MisskeyLogger from '@/logger.js'; +import type MisskeyLogger from '@/logger.js'; +import type { Data } from '@/logger.js'; +import type { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import { MiAbuseUserReport } from '@/models/AbuseUserReport.js'; @@ -95,12 +97,6 @@ import { SkApInboxLog } from '@/models/SkApInboxLog.js'; pg.types.setTypeParser(20, Number); -export const dbLogger = new MisskeyLogger('db'); - -const sqlLogger = dbLogger.createSubLogger('sql', 'gray'); -const sqlMigrateLogger = sqlLogger.createSubLogger('migrate'); -const sqlSchemaLogger = sqlLogger.createSubLogger('schema'); - export type LoggerProps = { disableQueryTruncation?: boolean; enableQueryLogging?: boolean; @@ -118,16 +114,26 @@ function truncateSql(sql: string) { return sql.length > 100 ? `${sql.substring(0, 100)} [truncated]` : sql; } -function stringifyParameter(param: any) { +function stringifyParameter(param: unknown): string { if (param instanceof Date) { return param.toISOString(); } else { - return param; + return String(param); } } -class MyCustomLogger implements Logger { - constructor(private props: LoggerProps = {}) { +class TypeORMLogger implements Logger { + private readonly sqlLogger: MisskeyLogger; + private readonly sqlMigrateLogger: MisskeyLogger; + private readonly sqlSchemaLogger: MisskeyLogger; + + constructor( + private readonly props: LoggerProps = {}, + dbLogger: MisskeyLogger, + ) { + this.sqlLogger = dbLogger.createSubLogger('sql', 'gray'); + this.sqlMigrateLogger = this.sqlLogger.createSubLogger('migrate'); + this.sqlSchemaLogger = this.sqlLogger.createSubLogger('schema'); } @bindThis @@ -143,9 +149,9 @@ class MyCustomLogger implements Logger { } @bindThis - private transformParameters(parameters?: any[]) { + private transformParameters(parameters?: unknown[]): Data | undefined { if (this.props.enableQueryParamLogging && parameters && parameters.length > 0) { - return parameters.reduce((params, p, i) => { + return parameters.reduce>((params: Record, p, i) => { params[`$${i + 1}`] = stringifyParameter(p); return params; }, {} as Record); @@ -155,37 +161,37 @@ class MyCustomLogger implements Logger { } @bindThis - public logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { + public logQuery(query: string, parameters?: unknown[], queryRunner?: QueryRunner) { if (!this.props.enableQueryLogging) return; const prefix = (this.props.printReplicationMode && queryRunner) ? `[${queryRunner.getReplicationMode()}] ` : undefined; const transformed = this.transformQueryLog(query, { prefix }); - sqlLogger.debug(`Query run: ${transformed}`, this.transformParameters(parameters)); + this.sqlLogger.debug(`Query run: ${transformed}`, this.transformParameters(parameters)); } @bindThis - public logQueryError(error: string, query: string, parameters?: any[], queryRunner?: QueryRunner) { + public logQueryError(error: string, query: string, parameters?: unknown[], queryRunner?: QueryRunner) { const prefix = (this.props.printReplicationMode && queryRunner) ? `[${queryRunner.getReplicationMode()}] ` : undefined; const transformed = this.transformQueryLog(query, { prefix }); - sqlLogger.error(`Query error (${error}): ${transformed}`, this.transformParameters(parameters)); + this.sqlLogger.error(`Query error (${error}): ${transformed}`, this.transformParameters(parameters)); } @bindThis - public logQuerySlow(time: number, query: string, parameters?: any[], queryRunner?: QueryRunner) { + public logQuerySlow(time: number, query: string, parameters?: unknown[], queryRunner?: QueryRunner) { const prefix = (this.props.printReplicationMode && queryRunner) ? `[${queryRunner.getReplicationMode()}] ` : undefined; const transformed = this.transformQueryLog(query, { prefix }); - sqlLogger.warn(`Query is slow (${time}ms): ${transformed}`, this.transformParameters(parameters)); + this.sqlLogger.warn(`Query is slow (${time}ms): ${transformed}`, this.transformParameters(parameters)); } @bindThis public logSchemaBuild(message: string) { - sqlSchemaLogger.debug(message); + this.sqlSchemaLogger.debug(message); } @bindThis @@ -193,18 +199,18 @@ class MyCustomLogger implements Logger { switch (level) { case 'log': case 'info': { - sqlLogger.info(message); + this.sqlLogger.info(message); break; } case 'warn': { - sqlLogger.warn(message); + this.sqlLogger.warn(message); } } } @bindThis public logMigration(message: string) { - sqlMigrateLogger.debug(message); + this.sqlMigrateLogger.debug(message); } } @@ -294,7 +300,8 @@ export const entities = [ const log = process.env.NODE_ENV !== 'production'; -export function createPostgresDataSource(config: Config) { +export function createPostgresDataSource(config: Config, loggerService: LoggerService) { + const dbLogger = loggerService.getLogger('db'); return new DataSource({ type: 'postgres', host: config.db.host, @@ -315,13 +322,13 @@ export function createPostgresDataSource(config: Config) { password: config.db.pass, database: config.db.db, }, - slaves: config.dbSlaves!.map(rep => ({ + slaves: config.dbSlaves?.map(rep => ({ host: rep.host, port: rep.port, username: rep.user, password: rep.pass, database: rep.db, - })), + })) ?? [], }, } : {}), synchronize: process.env.NODE_ENV === 'test', @@ -334,12 +341,12 @@ export function createPostgresDataSource(config: Config) { }, } : false, logging: log, - logger: new MyCustomLogger({ + logger: new TypeORMLogger({ disableQueryTruncation: config.logging?.sql?.disableQueryTruncation, enableQueryLogging: log, enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging, printReplicationMode: !!config.dbReplications, - }), + }, dbLogger), maxQueryExecutionTime: config.db.slowQueryThreshold, entities: entities, migrations: ['../../migration/*.js'], diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index 51fd97dc97..cb26a06529 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -47,7 +47,6 @@ import { ScheduleNotePostProcessorService } from './processors/ScheduleNotePostP @Module({ imports: [ - GlobalModule, CoreModule, ], providers: [ diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 76a617f027..35dc812652 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -11,7 +11,9 @@ import { DI } from '@/di-symbols.js'; import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { CheckModeratorsActivityProcessorService } from '@/queue/processors/CheckModeratorsActivityProcessorService.js'; +import { TimeService } from '@/global/TimeService.js'; import { renderFullError } from '@/misc/render-full-error.js'; +import { renderInlineError } from '@/misc/render-inline-error.js'; import { UserWebhookDeliverProcessorService } from './processors/UserWebhookDeliverProcessorService.js'; import { SystemWebhookDeliverProcessorService } from './processors/SystemWebhookDeliverProcessorService.js'; import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js'; @@ -60,10 +62,10 @@ function httpRelatedBackoff(attemptsMade: number) { return backoff; } -function getJobInfo(job: Bull.Job | undefined, increment = false): string { +function _getJobInfo(now: number, job: Bull.Job | undefined, increment = false): string { if (job == null) return '-'; - const age = Date.now() - job.timestamp; + const age = now - job.timestamp; const formated = age > 60000 ? `${Math.floor(age / 1000 / 60)}m` : age > 10000 ? `${Math.floor(age / 1000)}s` @@ -133,9 +135,15 @@ export class QueueProcessorService implements OnApplicationShutdown { private checkModeratorsActivityProcessorService: CheckModeratorsActivityProcessorService, private cleanProcessorService: CleanProcessorService, private scheduleNotePostProcessorService: ScheduleNotePostProcessorService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger; + // This is just to avoid modifying all the existing code. + const getJobInfo = (job: Bull.Job | undefined, increment = false) => { + return _getJobInfo(this.timeService.now, job, increment); + }; + //#region system { const processer = (job: Bull.Job) => { @@ -558,7 +566,7 @@ export class QueueProcessorService implements OnApplicationShutdown { // Render job if (job) { parts.push('job ['); - parts.push(getJobInfo(job)); + parts.push(_getJobInfo(this.timeService.now, job)); parts.push('] failed: '); } else { parts.push('job failed: '); @@ -596,7 +604,7 @@ export class QueueProcessorService implements OnApplicationShutdown { @bindThis public async stop(): Promise { - await Promise.all([ + await Promise.allSettled([ this.systemQueueWorker.close(), this.dbQueueWorker.close(), this.deliverQueueWorker.close(), @@ -607,7 +615,13 @@ export class QueueProcessorService implements OnApplicationShutdown { this.objectStorageQueueWorker.close(), this.endedPollNotificationQueueWorker.close(), this.schedulerNotePostQueueWorker.close(), - ]); + ]).then(res => { + for (const result of res) { + if (result.status === 'rejected') { + this.logger.error(`Error closing queue: ${renderInlineError(result.reason)}`); + } + } + }); } @bindThis diff --git a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts index 30bdd6ccca..6f71b42fa9 100644 --- a/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts +++ b/packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts @@ -11,6 +11,7 @@ import { bindThis } from '@/decorators.js'; import type { RetentionAggregationsRepository, UsersRepository } from '@/models/_.js'; import { deepClone } from '@/misc/clone.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -28,6 +29,7 @@ export class AggregateRetentionProcessorService { private idService: IdService, private queueLoggerService: QueueLoggerService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('aggregate-retention'); } @@ -36,18 +38,18 @@ export class AggregateRetentionProcessorService { public async process(): Promise { this.logger.info('Aggregating retention...'); - const now = new Date(); + const now = this.timeService.date; const dateKey = `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}`; // 過去(だいたい)30日分のレコードを取得 const pastRecords = await this.retentionAggregationsRepository.findBy({ - createdAt: MoreThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 31))), + createdAt: MoreThan(new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 31))), }); // 今日登録したユーザーを全て取得 const targetUsers = await this.usersRepository.findBy({ host: IsNull(), - id: MoreThan(this.idService.gen(Date.now() - (1000 * 60 * 60 * 24))), + id: MoreThan(this.idService.gen(this.timeService.now - (1000 * 60 * 60 * 24))), }); const targetUserIds = targetUsers.map(u => u.id); @@ -71,7 +73,7 @@ export class AggregateRetentionProcessorService { // 今日活動したユーザーを全て取得 const activeUsers = await this.usersRepository.findBy({ host: IsNull(), - lastActiveDate: MoreThan(new Date(Date.now() - (1000 * 60 * 60 * 24))), + lastActiveDate: MoreThan(new Date(this.timeService.now - (1000 * 60 * 60 * 24))), }); const activeUsersIds = activeUsers.map(u => u.id); diff --git a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts index 76d0cb4304..0471e27c0e 100644 --- a/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts +++ b/packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts @@ -10,6 +10,7 @@ import type { MutingsRepository } from '@/models/_.js'; import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { UserMutingService } from '@/core/UserMutingService.js'; +import { TimeService } from '@/global/TimeService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -23,6 +24,7 @@ export class CheckExpiredMutingsProcessorService { private userMutingService: UserMutingService, private queueLoggerService: QueueLoggerService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('check-expired-mutings'); } @@ -33,7 +35,7 @@ export class CheckExpiredMutingsProcessorService { const expired = await this.mutingsRepository.createQueryBuilder('muting') .where('muting.expiresAt IS NOT NULL') - .andWhere('muting.expiresAt < :now', { now: new Date() }) + .andWhere('muting.expiresAt < :now', { now: this.timeService.date }) .innerJoinAndSelect('muting.mutee', 'mutee') .getMany(); diff --git a/packages/backend/src/queue/processors/CheckModeratorsActivityProcessorService.ts b/packages/backend/src/queue/processors/CheckModeratorsActivityProcessorService.ts index 7821cd3d1d..5c94ab4e94 100644 --- a/packages/backend/src/queue/processors/CheckModeratorsActivityProcessorService.ts +++ b/packages/backend/src/queue/processors/CheckModeratorsActivityProcessorService.ts @@ -14,6 +14,7 @@ import { MiUser, type UserProfilesRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { SystemWebhookService } from '@/core/SystemWebhookService.js'; import { AnnouncementService } from '@/core/AnnouncementService.js'; +import { TimeService } from '@/global/TimeService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; // モデレーターが不在と判断する日付の閾値 @@ -92,6 +93,7 @@ export class CheckModeratorsActivityProcessorService { private announcementService: AnnouncementService, private systemWebhookService: SystemWebhookService, private queueLoggerService: QueueLoggerService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('check-moderators-activity'); } @@ -163,7 +165,7 @@ export class CheckModeratorsActivityProcessorService { */ @bindThis public async evaluateModeratorsInactiveDays(): Promise { - const today = new Date(); + const today = this.timeService.date; const inactivePeriod = new Date(today); inactivePeriod.setDate(today.getDate() - MODERATOR_INACTIVITY_LIMIT_DAYS); diff --git a/packages/backend/src/queue/processors/CleanProcessorService.ts b/packages/backend/src/queue/processors/CleanProcessorService.ts index 104d19103f..149d72de6a 100644 --- a/packages/backend/src/queue/processors/CleanProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanProcessorService.ts @@ -12,6 +12,7 @@ import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; import type { Config } from '@/config.js'; import { ReversiService } from '@/core/ReversiService.js'; +import { TimeService } from '@/global/TimeService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -35,6 +36,7 @@ export class CleanProcessorService { private queueLoggerService: QueueLoggerService, private reversiService: ReversiService, private idService: IdService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('clean'); } @@ -44,13 +46,13 @@ export class CleanProcessorService { this.logger.info('Cleaning...'); this.userIpsRepository.delete({ - createdAt: LessThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 90))), + createdAt: LessThan(new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 90))), }); // 使われてないアンテナを停止 if (this.config.deactivateAntennaThreshold > 0) { this.antennasRepository.update({ - lastUsedAt: LessThan(new Date(Date.now() - this.config.deactivateAntennaThreshold)), + lastUsedAt: LessThan(new Date(this.timeService.now - this.config.deactivateAntennaThreshold)), }, { isActive: false, }); @@ -58,7 +60,7 @@ export class CleanProcessorService { const expiredRoleAssignments = await this.roleAssignmentsRepository.createQueryBuilder('assign') .where('assign.expiresAt IS NOT NULL') - .andWhere('assign.expiresAt < :now', { now: new Date() }) + .andWhere('assign.expiresAt < :now', { now: this.timeService.date }) .getMany(); if (expiredRoleAssignments.length > 0) { diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 354f351358..993e00bcb9 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -10,11 +10,12 @@ import type { MiDriveFile, DriveFilesRepository } from '@/models/_.js'; import { MiUser } from '@/models/_.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; +import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { CleanRemoteFilesJobData } from '../types.js'; -import { IdService } from '@/core/IdService.js'; @Injectable() export class CleanRemoteFilesProcessorService { @@ -27,6 +28,7 @@ export class CleanRemoteFilesProcessorService { private driveService: DriveService, private queueLoggerService: QueueLoggerService, private idService: IdService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('clean-remote-files'); } @@ -35,7 +37,7 @@ export class CleanRemoteFilesProcessorService { public async process(job: Bull.Job): Promise { this.logger.info('Deleting cached remote files...'); - const olderThanTimestamp = Date.now() - (job.data.olderThanSeconds ?? 0) * 1000; + const olderThanTimestamp = this.timeService.now - (job.data.olderThanSeconds ?? 0) * 1000; const olderThanDate = new Date(olderThanTimestamp); const keepFilesInUse = job.data.keepFilesInUse ?? false; let deletedCount = 0; diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index 5bf64e4f04..06a4b7ab7f 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -19,7 +19,9 @@ import { ApLogService } from '@/core/ApLogService.js'; import { ReactionService } from '@/core/ReactionService.js'; import { QueueService } from '@/core/QueueService.js'; import { CacheService } from '@/core/CacheService.js'; -import { QueueLoggerService } from '../QueueLoggerService.js'; +import { QueueLoggerService } from '@/queue/QueueLoggerService.js'; +import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; +import * as Acct from '@/misc/acct.js'; import type * as Bull from 'bullmq'; import type { DbUserDeleteJobData } from '../types.js'; @@ -96,6 +98,7 @@ export class DeleteAccountProcessorService { private reactionService: ReactionService, private readonly apLogService: ApLogService, private readonly cacheService: CacheService, + private readonly apPersonService: ApPersonService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('delete-account'); } @@ -152,10 +155,14 @@ export class DeleteAccountProcessorService { await this.cacheService.hibernatedUserCache.delete(user.id); await this.cacheService.renoteMutingsCache.delete(user.id); await this.cacheService.userProfileCache.delete(user.id); - this.cacheService.userByIdCache.delete(user.id); - this.cacheService.localUserByIdCache.delete(user.id); + await this.cacheService.userByIdCache.delete(user.id); + await this.cacheService.userByAcctCache.delete(Acct.toString({ username: user.usernameLower, host: user.host })); + await this.cacheService.userFollowStatsCache.delete(user.id); if (user.token) { - this.cacheService.localUserByNativeTokenCache.delete(user.token); + await this.cacheService.nativeTokenCache.delete(user.token); + } + if (user.uri) { + await this.apPersonService.uriPersonCache.delete(user.uri); } await this.followingsRepository.delete({ diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index c54c27d661..0b1ef03a7a 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -19,6 +19,7 @@ import ApRequestChart from '@/core/chart/charts/ap-request.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { DeliverJobData } from '../types.js'; @@ -26,8 +27,6 @@ import type { DeliverJobData } from '../types.js'; @Injectable() export class DeliverProcessorService { private logger: Logger; - private suspendedHostsCache: MemorySingleCache; - private latest: string | null; constructor( @Inject(DI.meta) @@ -44,9 +43,9 @@ export class DeliverProcessorService { private apRequestChart: ApRequestChart, private federationChart: FederationChart, private queueLoggerService: QueueLoggerService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); - this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); // 1h } @bindThis @@ -57,25 +56,15 @@ export class DeliverProcessorService { return 'skip (blocked)'; } - // isSuspendedなら中断 - let suspendedHosts = this.suspendedHostsCache.get(); - if (suspendedHosts == null) { - suspendedHosts = await this.instancesRepository.find({ - where: { - suspensionState: Not('none'), - }, - }); - this.suspendedHostsCache.set(suspendedHosts); - } - if (suspendedHosts.map(x => x.host).includes(this.utilityService.toPuny(host))) { + const i = await this.federatedInstanceService.federatedInstanceCache.fetch(host); + if (i.suspensionState !== 'none') { return 'skip (suspended)'; } - const i = await (this.meta.enableStatsForFederatedInstances - ? this.federatedInstanceService.fetchOrRegister(host) - : this.federatedInstanceService.fetch(host)); + // Make sure info is up-to-date. + await this.fetchInstanceMetadataService.fetchInstanceMetadata(i); - // suspend server by software + // suspend server by software. if (i != null && this.utilityService.isDeliverSuspendedSoftware(i)) { return 'skip (software suspended)'; } @@ -97,10 +86,6 @@ export class DeliverProcessorService { }); } - if (this.meta.enableStatsForFederatedInstances) { - this.fetchInstanceMetadataService.fetchInstanceMetadata(i); - } - if (this.meta.enableChartsForFederatedInstances) { this.instanceChart.requestSent(i.host, true); } @@ -116,11 +101,11 @@ export class DeliverProcessorService { if (!i.isNotResponding) { this.federatedInstanceService.update(i.id, { isNotResponding: true, - notRespondingSince: new Date(), + notRespondingSince: this.timeService.date, }); } else if (i.notRespondingSince) { // 1週間以上不通ならサスペンド - if (i.suspensionState === 'none' && i.notRespondingSince.getTime() <= Date.now() - 1000 * 60 * 60 * 24 * 7) { + if (i.suspensionState === 'none' && i.notRespondingSince.getTime() <= this.timeService.now - 1000 * 60 * 60 * 24 * 7) { this.federatedInstanceService.update(i.id, { suspensionState: 'autoSuspendedForNotResponding', }); @@ -129,7 +114,7 @@ export class DeliverProcessorService { // isNotRespondingがtrueでnotRespondingSinceがnullの場合はnotRespondingSinceをセット // notRespondingSinceは新たな機能なので、それ以前のデータにはnotRespondingSinceがない場合がある this.federatedInstanceService.update(i.id, { - notRespondingSince: new Date(), + notRespondingSince: this.timeService.date, }); } diff --git a/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts b/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts index 58d542635f..97b006755f 100644 --- a/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAccountDataProcessorService.ts @@ -22,6 +22,7 @@ import { Packed } from '@/misc/json-schema.js'; import { UtilityService } from '@/core/UtilityService.js'; import { DownloadService } from '@/core/DownloadService.js'; import { EmailService } from '@/core/EmailService.js'; +import { TimeService } from '@/global/TimeService.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -80,6 +81,7 @@ export class ExportAccountDataProcessorService { private downloadService: DownloadService, private emailService: EmailService, private queueLoggerService: QueueLoggerService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-account-data'); } @@ -125,7 +127,7 @@ export class ExportAccountDataProcessorService { }); }; - await writeUser(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","user":[`); + await writeUser(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","user":[`); // eslint-disable-next-line @typescript-eslint/no-unused-vars const { host, uri, sharedInbox, followersUri, lastFetchedAt, inbox, ...userTrimmed } = user; @@ -160,7 +162,7 @@ export class ExportAccountDataProcessorService { // eslint-disable-next-line @typescript-eslint/no-unused-vars const { emailVerifyCode, twoFactorBackupSecret, twoFactorSecret, password, twoFactorTempSecret, userHost, ...profileTrimmed } = profile; - await writeProfile(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","profile":[`); + await writeProfile(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","profile":[`); await writeProfile(JSON.stringify(profileTrimmed)); @@ -191,7 +193,7 @@ export class ExportAccountDataProcessorService { }); }; - await writeIPs(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","ips":[`); + await writeIPs(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","ips":[`); for (const signin of signins) { // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -226,7 +228,7 @@ export class ExportAccountDataProcessorService { }); }; - await writeNotes(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","notes":[`); + await writeNotes(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","notes":[`); let noteCursor: MiNote['id'] | null = null; let exportedNotesCount = 0; @@ -287,7 +289,7 @@ export class ExportAccountDataProcessorService { }); }; - await writeFollowing(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","followings":[`); + await writeFollowing(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","followings":[`); let followingsCursor: MiFollowing['id'] | null = null; let exportedFollowingsCount = 0; @@ -321,7 +323,7 @@ export class ExportAccountDataProcessorService { continue; } - if (u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) { + if (u.updatedAt && (this.timeService.now - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) { continue; } @@ -357,7 +359,7 @@ export class ExportAccountDataProcessorService { }); }; - await writeFollowers(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","followers":[`); + await writeFollowers(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","followers":[`); let followersCursor: MiFollowing['id'] | null = null; let exportedFollowersCount = 0; @@ -420,7 +422,7 @@ export class ExportAccountDataProcessorService { fs.mkdirSync(`${path}/files`); - await writeDrive(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","drive":[`); + await writeDrive(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","drive":[`); const driveFiles = await this.driveFilesRepository.find({ where: { userId: user.id } }); @@ -476,7 +478,7 @@ export class ExportAccountDataProcessorService { }); }; - await writeMuting(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","mutings":[`); + await writeMuting(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","mutings":[`); let exportedMutingCount = 0; let mutingCursor: MiMuting['id'] | null = null; @@ -539,7 +541,7 @@ export class ExportAccountDataProcessorService { }); }; - await writeBlocking(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","blockings":[`); + await writeBlocking(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","blockings":[`); let exportedBlockingCount = 0; let blockingCursor: MiBlocking['id'] | null = null; @@ -601,7 +603,7 @@ export class ExportAccountDataProcessorService { }); }; - await writeFavorite(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","favorites":[`); + await writeFavorite(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","favorites":[`); let exportedFavoritesCount = 0; let favoriteCursor: MiNoteFavorite['id'] | null = null; @@ -662,7 +664,7 @@ export class ExportAccountDataProcessorService { }); }; - await writeAntenna(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","antennas":[`); + await writeAntenna(`{"metaVersion":1,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","antennas":[`); const antennas = await this.antennasRepository.findBy({ userId: user.id }); @@ -749,7 +751,7 @@ export class ExportAccountDataProcessorService { archiveStream.on('close', async () => { this.logger.debug(`Exported to path: ${archivePath}`); - const fileName = 'data-request-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; + const fileName = 'data-request-' + dateFormat(this.timeService.date, 'yyyy-MM-dd-HH-mm-ss') + '.zip'; const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); this.logger.debug(`Exported to drive: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts index 61d76da5ac..352dfe5cd5 100644 --- a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts @@ -15,6 +15,7 @@ import { bindThis } from '@/decorators.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; import { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { DBExportAntennasData } from '../types.js'; import type * as Bull from 'bullmq'; @@ -37,6 +38,7 @@ export class ExportAntennasProcessorService { private utilityService: UtilityService, private queueLoggerService: QueueLoggerService, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-antennas'); } @@ -98,7 +100,7 @@ export class ExportAntennasProcessorService { write(']'); stream.end(); - const fileName = 'antennas-' + DateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; + const fileName = 'antennas-' + DateFormat(this.timeService.date, 'yyyy-MM-dd-HH-mm-ss') + '.json'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' }); this.logger.debug('Exported to: ' + driveFile.id); diff --git a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts index 4c17c3f718..9323d9b41f 100644 --- a/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportBlockingProcessorService.ts @@ -14,6 +14,7 @@ import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; import { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -34,6 +35,7 @@ export class ExportBlockingProcessorService { private notificationService: NotificationService, private driveService: DriveService, private queueLoggerService: QueueLoggerService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-blocking'); } @@ -108,7 +110,7 @@ export class ExportBlockingProcessorService { stream.end(); this.logger.debug(`Exported to: ${path}`); - const fileName = 'blocking-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const fileName = 'blocking-' + dateFormat(this.timeService.date, 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' }); this.logger.debug(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/ExportClipsProcessorService.ts b/packages/backend/src/queue/processors/ExportClipsProcessorService.ts index 1d34d2b4e6..3036de5125 100644 --- a/packages/backend/src/queue/processors/ExportClipsProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportClipsProcessorService.ts @@ -20,6 +20,7 @@ import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.j import { Packed } from '@/misc/json-schema.js'; import { IdService } from '@/core/IdService.js'; import { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @@ -45,6 +46,7 @@ export class ExportClipsProcessorService { private queueLoggerService: QueueLoggerService, private idService: IdService, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-clips'); } @@ -78,7 +80,7 @@ export class ExportClipsProcessorService { this.logger.debug(`Exported to: ${path}`); - const fileName = 'clips-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; + const fileName = 'clips-' + dateFormat(this.timeService.date, 'yyyy-MM-dd-HH-mm-ss') + '.json'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' }); this.logger.debug(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts index b8f208bbfc..474732804d 100644 --- a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts @@ -17,6 +17,7 @@ import { DriveService } from '@/core/DriveService.js'; import { createTemp, createTempDir } from '@/misc/create-temp.js'; import { DownloadService } from '@/core/DownloadService.js'; import { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -39,6 +40,7 @@ export class ExportCustomEmojisProcessorService { private downloadService: DownloadService, private queueLoggerService: QueueLoggerService, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-custom-emojis'); } @@ -76,7 +78,7 @@ export class ExportCustomEmojisProcessorService { }); }; - await writeMeta(`{"metaVersion":2,"host":"${this.config.host}","exportedAt":"${new Date().toString()}","emojis":[`); + await writeMeta(`{"metaVersion":2,"host":"${this.config.host}","exportedAt":"${this.timeService.date.toString()}","emojis":[`); const customEmojis = await this.emojisRepository.find({ where: { @@ -133,7 +135,7 @@ export class ExportCustomEmojisProcessorService { archiveStream.on('close', async () => { this.logger.debug(`Exported to: ${archivePath}`); - const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip'; + const fileName = 'custom-emojis-' + dateFormat(this.timeService.date, 'yyyy-MM-dd-HH-mm-ss') + '.zip'; const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true }); this.logger.debug(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts index b5716f2d49..3132577094 100644 --- a/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts @@ -17,6 +17,7 @@ import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; import { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @@ -39,6 +40,7 @@ export class ExportFavoritesProcessorService { private queueLoggerService: QueueLoggerService, private idService: IdService, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-favorites'); } @@ -122,7 +124,7 @@ export class ExportFavoritesProcessorService { stream.end(); this.logger.debug(`Exported to: ${path}`); - const fileName = 'favorites-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; + const fileName = 'favorites-' + dateFormat(this.timeService.date, 'yyyy-MM-dd-HH-mm-ss') + '.json'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' }); this.logger.debug(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts index 883f35e366..45f9965417 100644 --- a/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportFollowingProcessorService.ts @@ -15,6 +15,7 @@ import { createTemp } from '@/misc/create-temp.js'; import type { MiFollowing } from '@/models/Following.js'; import { UtilityService } from '@/core/UtilityService.js'; import { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -38,6 +39,7 @@ export class ExportFollowingProcessorService { private driveService: DriveService, private queueLoggerService: QueueLoggerService, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-following'); } @@ -91,7 +93,7 @@ export class ExportFollowingProcessorService { continue; } - if (job.data.excludeInactive && u.updatedAt && (Date.now() - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) { + if (job.data.excludeInactive && u.updatedAt && (this.timeService.now - u.updatedAt.getTime() > 1000 * 60 * 60 * 24 * 90)) { continue; } @@ -112,7 +114,7 @@ export class ExportFollowingProcessorService { stream.end(); this.logger.debug(`Exported to: ${path}`); - const fileName = 'following-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const fileName = 'following-' + dateFormat(this.timeService.date, 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' }); this.logger.debug(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts index 9cdb94beaf..bc5a311d70 100644 --- a/packages/backend/src/queue/processors/ExportMutingProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportMutingProcessorService.ts @@ -15,6 +15,7 @@ import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; import { NotificationService } from '@/core/NotificationService.js'; import { bindThis } from '@/decorators.js'; +import { TimeService } from '@/global/TimeService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @@ -34,6 +35,7 @@ export class ExportMutingProcessorService { private driveService: DriveService, private queueLoggerService: QueueLoggerService, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-muting'); } @@ -109,7 +111,7 @@ export class ExportMutingProcessorService { stream.end(); this.logger.debug(`Exported to: ${path}`); - const fileName = 'mute-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const fileName = 'mute-' + dateFormat(this.timeService.date, 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' }); this.logger.debug(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index 7d49a8dab2..8a6a13841e 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -21,6 +21,7 @@ import { IdService } from '@/core/IdService.js'; import { NotificationService } from '@/core/NotificationService.js'; import { JsonArrayStream } from '@/misc/JsonArrayStream.js'; import { FileWriterStream } from '@/misc/FileWriterStream.js'; +import { TimeService } from '@/global/TimeService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; @@ -114,6 +115,7 @@ export class ExportNotesProcessorService { private driveFileEntityService: DriveFileEntityService, private idService: IdService, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-notes'); } @@ -149,7 +151,7 @@ export class ExportNotesProcessorService { this.logger.debug(`Exported to: ${path}`); - const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; + const fileName = 'notes-' + dateFormat(this.timeService.date, 'yyyy-MM-dd-HH-mm-ss') + '.json'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' }); this.logger.debug(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts index 43043e3a26..39c71334a5 100644 --- a/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportUserListsProcessorService.ts @@ -14,6 +14,7 @@ import { DriveService } from '@/core/DriveService.js'; import { createTemp } from '@/misc/create-temp.js'; import { UtilityService } from '@/core/UtilityService.js'; import { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -37,6 +38,7 @@ export class ExportUserListsProcessorService { private driveService: DriveService, private queueLoggerService: QueueLoggerService, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('export-user-lists'); } @@ -88,7 +90,7 @@ export class ExportUserListsProcessorService { stream.end(); this.logger.debug(`Exported to: ${path}`); - const fileName = 'user-lists-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.csv'; + const fileName = 'user-lists-' + dateFormat(this.timeService.date, 'yyyy-MM-dd-HH-mm-ss') + '.csv'; const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'csv' }); this.logger.debug(`Exported to: ${driveFile.id}`); diff --git a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts index 0fca613f29..4a69c31200 100644 --- a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts @@ -12,6 +12,7 @@ import type { AntennasRepository, UsersRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { NotificationService } from '@/core/NotificationService.js'; +import { TimeService } from '@/global/TimeService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import { DBAntennaImportJobData } from '../types.js'; import type * as Bull from 'bullmq'; @@ -67,6 +68,7 @@ export class ImportAntennasProcessorService { private idService: IdService, private globalEventService: GlobalEventService, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('import-antennas'); } @@ -81,7 +83,7 @@ export class ImportAntennasProcessorService { this.logger.debug(`Importing antennas of ${job.data.user.id} ...`); - const now = new Date(); + const now = this.timeService.date; try { for (const antenna of job.data.antenna) { if (antenna.keywords.length === 0 || antenna.keywords[0].every(x => x === '')) continue; diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index d35f4ac6d9..301ae8d061 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -92,10 +92,11 @@ export class ImportCustomEmojisProcessorService { continue; } const emojiPath = outputPath + '/' + record.fileName; - await this.emojisRepository.delete({ - name: nameNfc, - host: IsNull(), - }); + + const existing = await this.customEmojiService.emojisByIdCache.fetchMaybe(nameNfc); + if (existing) { + await this.customEmojiService.delete(existing.id, job.data.user); + } try { const driveFile = await this.driveService.addFile({ diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index d326854b37..6f5e929673 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -397,7 +397,7 @@ export class ImportNotesProcessorService { const visibility = followers ? toot.cc.includes('https://www.w3.org/ns/activitystreams#Public') ? 'home' : 'followers' : 'public'; const date = new Date(toot.object.published); - let text = undefined; + let text: string | undefined = undefined; const files: MiDriveFile[] = []; let reply: MiNote | null = null; @@ -464,7 +464,7 @@ export class ImportNotesProcessorService { } const date = new Date(post.object.published); - let text = undefined; + let text: string | undefined = undefined; const files: MiDriveFile[] = []; let reply: MiNote | null = null; @@ -547,7 +547,7 @@ export class ImportNotesProcessorService { const files: MiDriveFile[] = []; function decodeIGString(str: string) { - const arr = []; + const arr: number[] = []; for (let i = 0; i < str.length; i++) { arr.push(str.charCodeAt(i)); } @@ -700,7 +700,7 @@ export class ImportNotesProcessorService { const files: MiDriveFile[] = []; function decodeFBString(str: string) { - const arr = []; + const arr: number[] = []; for (let i = 0; i < str.length; i++) { arr.push(str.charCodeAt(i)); } @@ -708,7 +708,7 @@ export class ImportNotesProcessorService { } if (post.attachments && this.isIterable(post.attachments)) { - const media = []; + const media: any[] = []; for await (const data of post.attachments[0].data) { if (data.media) { media.push(data.media); diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 5f82d558b3..4efbb088c6 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -31,6 +31,7 @@ import { SkApInboxLog } from '@/models/_.js'; import type { Config } from '@/config.js'; import { ApLogService, calculateDurationSince } from '@/core/ApLogService.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; +import { TimeService } from '@/global/TimeService.js'; import { isRetryableError } from '@/misc/is-retryable-error.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; @@ -66,6 +67,7 @@ export class InboxProcessorService implements OnApplicationShutdown { private queueLoggerService: QueueLoggerService, private readonly apLogService: ApLogService, private readonly updateInstanceQueue: UpdateInstanceQueue, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('inbox'); } @@ -263,7 +265,7 @@ export class InboxProcessorService implements OnApplicationShutdown { if (i == null) return; this.updateInstanceQueue.enqueue(i.id, { - latestRequestReceivedAt: new Date(), + latestRequestReceivedAt: this.timeService.date, shouldUnsuspend: i.suspensionState === 'autoSuspendedForNotResponding', }); @@ -322,7 +324,7 @@ export class InboxProcessorService implements OnApplicationShutdown { @bindThis public async performUpdateInstance(id: string, job: UpdateInstanceJob) { await this.federatedInstanceService.update(id, { - latestRequestReceivedAt: new Date(), + latestRequestReceivedAt: this.timeService.date, isNotResponding: false, // もしサーバーが死んでるために配信が止まっていた場合には自動的に復活させてあげる suspensionState: job.shouldUnsuspend ? 'none' : undefined, diff --git a/packages/backend/src/queue/processors/ScheduleNotePostProcessorService.ts b/packages/backend/src/queue/processors/ScheduleNotePostProcessorService.ts index 73088f3312..c550106e18 100644 --- a/packages/backend/src/queue/processors/ScheduleNotePostProcessorService.ts +++ b/packages/backend/src/queue/processors/ScheduleNotePostProcessorService.ts @@ -13,6 +13,7 @@ import { NotificationService } from '@/core/NotificationService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { MiScheduleNoteType } from '@/models/NoteSchedule.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; +import { TimeService } from '@/global/TimeService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { ScheduleNotePostJobData } from '../types.js'; @@ -37,6 +38,7 @@ export class ScheduleNotePostProcessorService { private noteCreateService: NoteCreateService, private queueLoggerService: QueueLoggerService, private notificationService: NotificationService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('schedule-note-post'); } @@ -118,7 +120,7 @@ export class ScheduleNotePostProcessorService { const createdNote = await this.noteCreateService.create(me, { ...note, - createdAt: new Date(), + createdAt: this.timeService.date, files, poll: note.poll ? { choices: note.poll.choices, diff --git a/packages/backend/src/queue/processors/SystemWebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/SystemWebhookDeliverProcessorService.ts index f9fcd1e928..641aec9fe8 100644 --- a/packages/backend/src/queue/processors/SystemWebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/SystemWebhookDeliverProcessorService.ts @@ -11,6 +11,7 @@ import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { StatusError } from '@/misc/status-error.js'; +import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; @@ -29,6 +30,7 @@ export class SystemWebhookDeliverProcessorService { private httpRequestService: HttpRequestService, private queueLoggerService: QueueLoggerService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('webhook'); } @@ -58,7 +60,7 @@ export class SystemWebhookDeliverProcessorService { }); this.systemWebhooksRepository.update({ id: job.data.webhookId }, { - latestSentAt: new Date(), + latestSentAt: this.timeService.date, latestStatus: res.status, }); @@ -67,7 +69,7 @@ export class SystemWebhookDeliverProcessorService { this.logger.error(`Failed to send webhook: ${renderInlineError(res)}`); this.systemWebhooksRepository.update({ id: job.data.webhookId }, { - latestSentAt: new Date(), + latestSentAt: this.timeService.date, latestStatus: res instanceof StatusError ? res.statusCode : 1, }); diff --git a/packages/backend/src/queue/processors/UserWebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/UserWebhookDeliverProcessorService.ts index 0208ce6038..6d96cfa3f4 100644 --- a/packages/backend/src/queue/processors/UserWebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/UserWebhookDeliverProcessorService.ts @@ -11,6 +11,7 @@ import type { Config } from '@/config.js'; import type Logger from '@/logger.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { StatusError } from '@/misc/status-error.js'; +import { TimeService } from '@/global/TimeService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import { UserWebhookDeliverJobData } from '../types.js'; @@ -28,6 +29,7 @@ export class UserWebhookDeliverProcessorService { private httpRequestService: HttpRequestService, private queueLoggerService: QueueLoggerService, + private readonly timeService: TimeService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('webhook'); } @@ -58,14 +60,14 @@ export class UserWebhookDeliverProcessorService { }); this.webhooksRepository.update({ id: job.data.webhookId }, { - latestSentAt: new Date(), + latestSentAt: this.timeService.date, latestStatus: res.status, }); return 'Success'; } catch (res) { this.webhooksRepository.update({ id: job.data.webhookId }, { - latestSentAt: new Date(), + latestSentAt: this.timeService.date, latestStatus: res instanceof StatusError ? res.statusCode : 1, }); diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 27d25d2152..e73de15241 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -21,6 +21,7 @@ import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import { QueueService } from '@/core/QueueService.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; +import { isLocalUser } from '@/models/User.js'; import { UserKeypairService } from '@/core/UserKeypairService.js'; import type { MiUserPublickey } from '@/models/UserPublickey.js'; import type { MiFollowing } from '@/models/Following.js'; @@ -28,7 +29,6 @@ import { countIf } from '@/misc/prelude/array.js'; import type { MiNote } from '@/models/Note.js'; import { QueryService } from '@/core/QueryService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; import type Logger from '@/logger.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; @@ -36,6 +36,7 @@ import { IActivity, IAnnounce, ICreate } from '@/core/activitypub/type.js'; import { isPureRenote, isQuote, isRenote } from '@/misc/is-renote.js'; import * as Acct from '@/misc/acct.js'; import { CacheService } from '@/core/CacheService.js'; +import { CustomEmojiService, encodeEmojiKey } from '@/core/CustomEmojiService.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions, FastifyBodyParser } from 'fastify'; import type { FindOptionsWhere } from 'typeorm'; import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; @@ -80,7 +81,6 @@ export class ActivityPubServerService { private followRequestsRepository: FollowRequestsRepository, private utilityService: UtilityService, - private userEntityService: UserEntityService, private apRendererService: ApRendererService, private apDbResolverService: ApDbResolverService, private queueService: QueueService, @@ -89,6 +89,7 @@ export class ActivityPubServerService { private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private loggerService: LoggerService, private readonly cacheService: CacheService, + private readonly customEmojiService: CustomEmojiService, ) { //this.createServer = this.createServer.bind(this); this.logger = this.loggerService.getLogger('apserv', 'pink'); @@ -974,7 +975,7 @@ export class ActivityPubServerService { const keypair = await this.userKeypairService.getUserKeypair(user.id); - if (this.userEntityService.isLocalUser(user)) { + if (isLocalUser(user)) { this.setResponseType(request, reply); return (this.apRendererService.addContext(this.apRendererService.renderKey(user, keypair))); } else { @@ -1037,10 +1038,8 @@ export class ActivityPubServerService { const { reject } = await this.checkAuthorizedFetch(request, reply); if (reject) return; - const emoji = await this.emojisRepository.findOneBy({ - host: IsNull(), - name: request.params.emoji, - }); + const emojiKey = encodeEmojiKey({ name: request.params.emoji, host: null }); + const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(emojiKey); if (emoji == null || emoji.localOnly) { reply.code(404); @@ -1048,7 +1047,7 @@ export class ActivityPubServerService { } this.setResponseType(request, reply); - return (this.apRendererService.addContext(await this.apRendererService.renderEmoji(emoji))); + return (this.apRendererService.addContext(this.apRendererService.renderEmoji(emoji))); }); // like diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index 4cc7653ea1..b6efefa468 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -4,9 +4,8 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { IsNull, MoreThan } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { UsersRepository } from '@/models/_.js'; +import type { MiMeta } from '@/models/Meta.js'; import type { Config } from '@/config.js'; import { MetaService } from '@/core/MetaService.js'; import { MemorySingleCache } from '@/misc/cache.js'; @@ -15,6 +14,7 @@ import NotesChart from '@/core/chart/charts/notes.js'; import UsersChart from '@/core/chart/charts/users.js'; import { DEFAULT_POLICIES } from '@/core/RoleService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; +import { InstanceStatsService } from '@/core/InstanceStatsService.js'; import type { FastifyInstance, FastifyPluginOptions } from 'fastify'; const nodeinfo2_1path = '/nodeinfo/2.1'; @@ -27,13 +27,14 @@ export class NodeinfoServerService { @Inject(DI.config) private config: Config, - @Inject(DI.usersRepository) - private usersRepository: UsersRepository, + @Inject(DI.meta) + private readonly meta: MiMeta, private systemAccountService: SystemAccountService, private metaService: MetaService, private notesChart: NotesChart, private usersChart: UsersChart, + private readonly instanceStatsService: InstanceStatsService, ) { //this.createServer = this.createServer.bind(this); } @@ -51,38 +52,28 @@ export class NodeinfoServerService { @bindThis public createServer(fastify: FastifyInstance, options: FastifyPluginOptions, done: (err?: Error) => void) { - const nodeinfo2 = async (version: number) => { - const now = Date.now(); - - const notesChart = await this.notesChart.getChart('hour', 1, null); - const localPosts = notesChart.local.total[0]; - - const usersChart = await this.usersChart.getChart('hour', 1, null); - const total = usersChart.local.total[0]; - - const [ - meta, - activeHalfyear, - activeMonth, - ] = await Promise.all([ - this.metaService.fetch(true), - // 重い - this.usersRepository.count({ where: { host: IsNull(), isBot: false, lastActiveDate: MoreThan(new Date(now - 15552000000)) } }), - this.usersRepository.count({ where: { host: IsNull(), isBot: false, lastActiveDate: MoreThan(new Date(now - 2592000000)) } }), - ]); - + const nodeinfo2 = async (version: '2.0' | '2.1') => { + const meta = this.meta; + const stats = await this.instanceStatsService.fetch(); const proxyAccount = await this.systemAccountService.fetch('proxy'); const basePolicies = { ...DEFAULT_POLICIES, ...meta.policies }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const document: any = { - software: { - name: 'sharkey', - version: this.config.version, - homepage: nodeinfo_homepage, + const software = { + name: 'sharkey', + version: this.config.version, + }; + + if (version !== '2.0') { + Object.assign(software, { + homepage: meta.repositoryUrl ?? nodeinfo_homepage, repository: meta.repositoryUrl, - }, + }); + } + + return { + version, + software, protocols: ['activitypub'], services: { inbound: [] as string[], @@ -90,8 +81,12 @@ export class NodeinfoServerService { }, openRegistrations: !meta.disableRegistration, usage: { - users: { total, activeHalfyear, activeMonth }, - localPosts, + users: { + total: stats.usersTotal, + activeHalfyear: stats.usersActiveSixMonths, + activeMonth: stats.usersActiveMonth, + }, + localPosts: stats.notesTotal, localComments: 0, }, metadata: { @@ -138,18 +133,9 @@ export class NodeinfoServerService { themeColor: meta.themeColor ?? '#86b300', }, }; - if (version >= 21) { - document.software.repository = meta.repositoryUrl; - document.software.homepage = meta.repositoryUrl; - } - return document; }; - const cache = new MemorySingleCache>>(1000 * 60 * 10); // 10m - fastify.get(nodeinfo2_1path, async (request, reply) => { - const base = await cache.fetch(() => nodeinfo2(21)); - reply .type( 'application/json; profile="http://nodeinfo.diaspora.software/ns/schema/2.1#"', @@ -159,14 +145,10 @@ export class NodeinfoServerService { .header('Access-Control-Allow-Methods', 'GET, OPTIONS') .header('Access-Control-Allow-Origin', '*') .header('Access-Control-Expose-Headers', 'Vary'); - return { version: '2.1', ...base }; + return await nodeinfo2('2.1'); }); fastify.get(nodeinfo2_0path, async (request, reply) => { - const base = await cache.fetch(() => nodeinfo2(20)); - - delete (base as any).software.repository; - reply .type( 'application/json; profile="http://nodeinfo.diaspora.software/ns/schema/2.0#"', @@ -176,7 +158,7 @@ export class NodeinfoServerService { .header('Access-Control-Allow-Methods', 'GET, OPTIONS') .header('Access-Control-Allow-Origin', '*') .header('Access-Control-Expose-Headers', 'Vary'); - return { version: '2.0', ...base }; + return await nodeinfo2('2.0'); }); done(); diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index caa82d1ce8..a25505f813 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -19,6 +19,7 @@ import type Logger from '@/logger.js'; import * as Acct from '@/misc/acct.js'; import { genIdenticon } from '@/misc/gen-identicon.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { CustomEmojiService, encodeEmojiKey } from '@/core/CustomEmojiService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; @@ -39,7 +40,7 @@ const _dirname = fileURLToPath(new URL('.', import.meta.url)); @Injectable() export class ServerService implements OnApplicationShutdown { private logger: Logger; - #fastify: FastifyInstance; + #fastify?: FastifyInstance; constructor( @Inject(DI.config) @@ -71,6 +72,7 @@ export class ServerService implements OnApplicationShutdown { private globalEventService: GlobalEventService, private loggerService: LoggerService, private oauth2ProviderService: OAuth2ProviderService, + private readonly customEmojiService: CustomEmojiService, ) { this.logger = this.loggerService.getLogger('server', 'gray'); } @@ -171,14 +173,15 @@ export class ServerService implements OnApplicationShutdown { return; } - const name = pathChunks.shift(); + const name = pathChunks.shift() as string; const host = pathChunks.pop(); - const emoji = await this.emojisRepository.findOneBy({ + const emojiKey = encodeEmojiKey({ // `@.` is the spec of ReactionService.decodeReaction - host: (host === undefined || host === '.') ? IsNull() : host, + host: (host === undefined || host === '.') ? null : host, name: name, }); + const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(emojiKey); reply.header('Content-Security-Policy', 'default-src \'none\'; style-src \'unsafe-inline\''); @@ -313,7 +316,7 @@ export class ServerService implements OnApplicationShutdown { await this.streamingApiServerService.detach(); this.logger.info('Disconnecting HTTP clients....;'); - await this.#fastify.close(); + await this.#fastify?.close(); this.logger.info('Server disposed.'); } @@ -322,6 +325,9 @@ export class ServerService implements OnApplicationShutdown { * Get the Fastify instance for testing. */ public get fastify(): FastifyInstance { + if (!this.#fastify) { + throw new Error('Cannot get fastify before starting server'); + } return this.#fastify; } diff --git a/packages/backend/src/server/SkRateLimiterService.ts b/packages/backend/src/server/SkRateLimiterService.ts index a53c58ba5a..212f00c86a 100644 --- a/packages/backend/src/server/SkRateLimiterService.ts +++ b/packages/backend/src/server/SkRateLimiterService.ts @@ -5,13 +5,14 @@ import { Inject, Injectable } from '@nestjs/common'; import Redis from 'ioredis'; -import type { TimeService } from '@/core/TimeService.js'; -import type { EnvService } from '@/core/EnvService.js'; -import { BucketRateLimit, LegacyRateLimit, LimitInfo, RateLimit, hasMinLimit, isLegacyRateLimit, Keyed, hasMaxLimit, disabledLimitInfo, MaxLegacyLimit, MinLegacyLimit } from '@/misc/rate-limit-utils.js'; -import { DI } from '@/di-symbols.js'; -import { MemoryKVCache } from '@/misc/cache.js'; import type { MiUser } from '@/models/_.js'; -import type { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; +import { EnvService } from '@/global/EnvService.js'; +import { BucketRateLimit, LegacyRateLimit, LimitInfo, RateLimit, hasMinLimit, isLegacyRateLimit, Keyed, hasMaxLimit, disabledLimitInfo, MaxLegacyLimit, MinLegacyLimit } from '@/misc/rate-limit-utils.js'; +import { RoleService } from '@/core/RoleService.js'; +import { CacheManagementService, type ManagedMemoryKVCache } from '@/global/CacheManagementService.js'; +import { ConflictError } from '@/misc/errors/ConflictError.js'; +import { DI } from '@/di-symbols.js'; // Sentinel value used for caching the default role template. // Required because MemoryKVCache doesn't support null keys. @@ -29,26 +30,24 @@ interface ParsedLimit { @Injectable() export class SkRateLimiterService { - // 1-minute cache interval - private readonly factorCache = new MemoryKVCache(1000 * 60); - // 10-second cache interval - private readonly lockoutCache = new MemoryKVCache(1000 * 10); + private readonly factorCache: ManagedMemoryKVCache; + private readonly lockoutCache: ManagedMemoryKVCache; private readonly requestCounts = new Map(); private readonly disabled: boolean; constructor( - @Inject('TimeService') - private readonly timeService: TimeService, @Inject(DI.redisForRateLimit) private readonly redisClient: Redis.Redis, - @Inject('RoleService') private readonly roleService: RoleService, + private readonly timeService: TimeService, - @Inject('EnvService') envService: EnvService, + cacheManagementService: CacheManagementService, ) { + this.factorCache = cacheManagementService.createMemoryKVCache('rateLimitFactor', 1000 * 60); // 1m + this.lockoutCache = cacheManagementService.createMemoryKVCache('rateLimitLockout', 1000 * 10); // 10s this.disabled = envService.env.NODE_ENV === 'test'; } @@ -389,8 +388,6 @@ function createLimitKey(limit: ParsedLimit, actor: string, value: string): strin return `rl_${actor}_${limit.key}_${value}`; } -export class ConflictError extends Error {} - interface LimitCounter { timestamp: number; counter: number; diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index ac3f7ae0f3..6605783ff1 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -20,6 +20,7 @@ import { RoleService } from '@/core/RoleService.js'; import type { Config } from '@/config.js'; import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; +import { TimeService, type TimerHandle } from '@/global/TimeService.js'; import { renderInlineError } from '@/misc/render-inline-error.js'; import { renderFullError } from '@/misc/render-full-error.js'; import { ApiError } from './error.js'; @@ -39,7 +40,7 @@ const accessDenied = { export class ApiCallService implements OnApplicationShutdown { private logger: Logger; private userIpHistories: Map>; - private userIpHistoriesClearIntervalId: NodeJS.Timeout; + private userIpHistoriesClearIntervalId: TimerHandle; constructor( @Inject(DI.meta) @@ -55,13 +56,14 @@ export class ApiCallService implements OnApplicationShutdown { private rateLimiterService: SkRateLimiterService, private roleService: RoleService, private apiLoggerService: ApiLoggerService, + private readonly timeService: TimeService, ) { this.logger = this.apiLoggerService.logger; this.userIpHistories = new Map>(); - this.userIpHistoriesClearIntervalId = setInterval(() => { + this.userIpHistoriesClearIntervalId = this.timeService.startTimer(() => { this.userIpHistories.clear(); - }, 1000 * 60 * 60); + }, 1000 * 60 * 60, { repeated: true }); } #sendApiError(reply: FastifyReply, err: ApiError): void { @@ -284,7 +286,7 @@ export class ApiCallService implements OnApplicationShutdown { try { this.userIpsRepository.createQueryBuilder().insert().values({ - createdAt: new Date(), + createdAt: this.timeService.date, userId: user.id, ip: ip, }).orIgnore(true).execute(); @@ -456,7 +458,7 @@ export class ApiCallService implements OnApplicationShutdown { @bindThis public dispose(): void { - clearInterval(this.userIpHistoriesClearIntervalId); + this.timeService.stopTimer(this.userIpHistoriesClearIntervalId); } @bindThis diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index 3681602e1f..2c5615c084 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -14,8 +14,13 @@ import { CacheService } from '@/core/CacheService.js'; import { isNativeUserToken } from '@/misc/token.js'; import { bindThis } from '@/decorators.js'; import { attachCallerId } from '@/misc/attach-caller-id.js'; +import { CacheManagementService, type ManagedMemoryKVCache } from '@/global/CacheManagementService.js'; +import { TimeService } from '@/global/TimeService.js'; export class AuthenticationError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + constructor(message: string) { super(message); this.name = 'AuthenticationError'; @@ -23,8 +28,8 @@ export class AuthenticationError extends Error { } @Injectable() -export class AuthenticateService implements OnApplicationShutdown { - private appCache: MemoryKVCache; +export class AuthenticateService { + private readonly appCache: ManagedMemoryKVCache; constructor( @Inject(DI.usersRepository) @@ -37,8 +42,11 @@ export class AuthenticateService implements OnApplicationShutdown { private appsRepository: AppsRepository, private cacheService: CacheService, + private readonly timeService: TimeService, + + cacheManagementService: CacheManagementService, ) { - this.appCache = new MemoryKVCache(1000 * 60 * 60 * 24 * 7); // 1w + this.appCache = cacheManagementService.createMemoryKVCache('app', 1000 * 60 * 60 * 24); // 1d } @bindThis @@ -48,8 +56,7 @@ export class AuthenticateService implements OnApplicationShutdown { } if (isNativeUserToken(token)) { - const user = await this.cacheService.localUserByNativeTokenCache.fetch(token, - () => this.usersRepository.findOneBy({ token }) as Promise); + const user = await this.cacheService.findOptionalLocalUserByNativeToken(token); if (user == null) { throw new AuthenticationError('user not found'); @@ -73,7 +80,7 @@ export class AuthenticateService implements OnApplicationShutdown { } this.accessTokensRepository.update(accessToken.id, { - lastUsedAt: new Date(), + lastUsedAt: this.timeService.date, }); // Loaded by relation above @@ -97,14 +104,4 @@ export class AuthenticateService implements OnApplicationShutdown { } } } - - @bindThis - public dispose(): void { - this.appCache.dispose(); - } - - @bindThis - public onApplicationShutdown(signal?: string | undefined): void { - this.dispose(); - } } diff --git a/packages/backend/src/server/api/GetterService.ts b/packages/backend/src/server/api/GetterService.ts index f2850e6258..edf70c0185 100644 --- a/packages/backend/src/server/api/GetterService.ts +++ b/packages/backend/src/server/api/GetterService.ts @@ -8,8 +8,9 @@ import { DI } from '@/di-symbols.js'; import type { NotesRepository, UsersRepository, NoteEditRepository } from '@/models/_.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; +import { isRemoteUser, isLocalUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { CacheService } from '@/core/CacheService.js'; import { bindThis } from '@/decorators.js'; @Injectable() @@ -24,7 +25,7 @@ export class GetterService { @Inject(DI.noteEditRepository) private noteEditRepository: NoteEditRepository, - private userEntityService: UserEntityService, + private readonly cacheService: CacheService, ) { } @@ -70,7 +71,7 @@ export class GetterService { */ @bindThis public async getUser(userId: MiUser['id']) { - const user = await this.usersRepository.findOneBy({ id: userId }); + const user = await this.cacheService.findOptionalUserById(userId); if (user == null) { throw new IdentifiableError('15348ddd-432d-49c2-8a5a-8069753becff', `User ${userId} does not exist`); @@ -86,8 +87,8 @@ export class GetterService { public async getRemoteUser(userId: MiUser['id']) { const user = await this.getUser(userId); - if (!this.userEntityService.isRemoteUser(user)) { - throw new Error('user is not a remote user'); + if (!isRemoteUser(user)) { + throw new IdentifiableError('aeac1339-2550-4521-a8e3-781f06d98656', 'User is not remote'); } return user; @@ -100,8 +101,8 @@ export class GetterService { public async getLocalUser(userId: MiUser['id']) { const user = await this.getUser(userId); - if (!this.userEntityService.isLocalUser(user)) { - throw new Error('user is not a local user'); + if (!isLocalUser(user)) { + throw new IdentifiableError('aeac1339-2550-4521-a8e3-781f06d98656', 'User is not local'); } return user; diff --git a/packages/backend/src/server/api/SignupApiService.ts b/packages/backend/src/server/api/SignupApiService.ts index 86896264dd..efaa059d36 100644 --- a/packages/backend/src/server/api/SignupApiService.ts +++ b/packages/backend/src/server/api/SignupApiService.ts @@ -21,6 +21,7 @@ import { L_CHARS, secureRndstr } from '@/misc/secure-rndstr.js'; import { RoleService } from '@/core/RoleService.js'; import Logger from '@/logger.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { TimeService } from '@/global/TimeService.js'; import { SigninService } from './SigninService.js'; import type { FastifyRequest, FastifyReply } from 'fastify'; @@ -60,6 +61,7 @@ export class SignupApiService { private emailService: EmailService, private roleService: RoleService, private loggerService: LoggerService, + private readonly timeService: TimeService, ) { this.logger = this.loggerService.getLogger('Signup'); } @@ -170,7 +172,7 @@ export class SignupApiService { return; } - if (ticket.expiresAt && ticket.expiresAt < new Date()) { + if (ticket.expiresAt && ticket.expiresAt < this.timeService.date) { reply.code(400); return; } @@ -184,7 +186,7 @@ export class SignupApiService { } // 認証しておらず、メール送信から30分以内ならエラー - if (ticket.usedAt && ticket.usedAt.getTime() + (1000 * 60 * 30) > Date.now()) { + if (ticket.usedAt && ticket.usedAt.getTime() + (1000 * 60 * 30) > this.timeService.now) { reply.code(400); return; } @@ -232,7 +234,7 @@ export class SignupApiService { if (ticket) { await this.registrationTicketsRepository.update(ticket.id, { - usedAt: new Date(), + usedAt: this.timeService.date, pendingUserId: pendingUser.id, }); } @@ -252,7 +254,7 @@ export class SignupApiService { if (ticket) { await this.registrationTicketsRepository.update(ticket.id, { - usedAt: new Date(), + usedAt: this.timeService.date, usedBy: account, usedById: account.id, }); @@ -289,7 +291,7 @@ export class SignupApiService { if (ticket) { await this.registrationTicketsRepository.update(ticket.id, { - usedAt: new Date(), + usedAt: this.timeService.date, usedBy: account, usedById: account.id, }); @@ -318,7 +320,7 @@ export class SignupApiService { try { const pendingUser = await this.userPendingsRepository.findOneByOrFail({ code }); - if (this.idService.parse(pendingUser.id).date.getTime() + (1000 * 60 * 30) < Date.now()) { + if (this.idService.parse(pendingUser.id).date.getTime() + (1000 * 60 * 30) < this.timeService.now) { throw new FastifyReplyError(400, 'EXPIRED'); } @@ -390,7 +392,7 @@ export class SignupApiService { private logIp(ip: string, ipDate: Date | null, userId: MiLocalUser['id']) { try { this.userIpsRepository.createQueryBuilder().insert().values({ - createdAt: ipDate ?? new Date(), + createdAt: ipDate ?? this.timeService.date, userId, ip, }).orIgnore(true).execute(); diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index f39ef51b7e..20d8fc4ca2 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -24,6 +24,7 @@ import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { QueryService } from '@/core/QueryService.js'; +import { TimeService, type TimerHandle } from '@/global/TimeService.js'; import { AuthenticateService, AuthenticationError } from './AuthenticateService.js'; import MainStreamConnection from './stream/Connection.js'; import { ChannelsService } from './stream/ChannelsService.js'; @@ -38,7 +39,7 @@ export class StreamingApiServerService implements OnApplicationShutdown { #wss?: WebSocket.WebSocketServer; #connections = new Map(); #connectionsByClient = new Map>(); // key: IP / user ID -> value: connection - #cleanConnectionsIntervalId: NodeJS.Timeout | null = null; + #cleanConnectionsIntervalId: TimerHandle | null = null; readonly #globalEv = new EventEmitter(); #logger: Logger; @@ -58,7 +59,9 @@ export class StreamingApiServerService implements OnApplicationShutdown { @Inject(DI.noteFavoritesRepository) private readonly noteFavoritesRepository: NoteFavoritesRepository, - private readonly queryService: QueryService, + @Inject(DI.config) + private config: Config, + private cacheService: CacheService, private authenticateService: AuthenticateService, private channelsService: ChannelsService, @@ -67,9 +70,8 @@ export class StreamingApiServerService implements OnApplicationShutdown { private channelFollowingService: ChannelFollowingService, private rateLimiterService: SkRateLimiterService, private loggerService: LoggerService, - - @Inject(DI.config) - private config: Config, + private readonly queryService: QueryService, + private readonly timeService: TimeService, ) { this.redisForSub.on('message', this.onRedis); this.#logger = loggerService.getLogger('streaming', 'coral'); @@ -205,6 +207,7 @@ export class StreamingApiServerService implements OnApplicationShutdown { this.notificationService, this.cacheService, this.channelFollowingService, + this.timeService, this.loggerService, user, app, requestIp, rateLimiter, @@ -265,17 +268,17 @@ export class StreamingApiServerService implements OnApplicationShutdown { await stream.listen(ev, connection); - this.#connections.set(connection, Date.now()); + this.#connections.set(connection, this.timeService.now); // TODO use collapsed queue - const userUpdateIntervalId = user ? setInterval(() => { + const userUpdateIntervalId = user ? this.timeService.startTimer(() => { this.usersService.updateLastActiveDate(user); - }, 1000 * 60 * 5) : null; + }, 1000 * 60 * 5, { repeated: true }) : null; if (user) { this.usersService.updateLastActiveDate(user); } const pong = () => { - this.#connections.set(connection, Date.now()); + this.#connections.set(connection, this.timeService.now); }; connection.once('close', () => { @@ -285,7 +288,7 @@ export class StreamingApiServerService implements OnApplicationShutdown { stream.dispose(); this.#globalEv.off('message', onRedisMessage); this.#connections.delete(connection); - if (userUpdateIntervalId) clearInterval(userUpdateIntervalId); + if (userUpdateIntervalId) this.timeService.stopTimer(userUpdateIntervalId); }); connection.on('error', this.onWsError); @@ -293,8 +296,8 @@ export class StreamingApiServerService implements OnApplicationShutdown { }); // 一定期間通信が無いコネクションは実際には切断されている可能性があるため定期的にterminateする - this.#cleanConnectionsIntervalId = setInterval(() => { - const now = Date.now(); + this.#cleanConnectionsIntervalId = this.timeService.startTimer(() => { + const now = this.timeService.now; for (const [connection, lastActive] of this.#connections.entries()) { if (now - lastActive > 1000 * 60 * 2) { connection.terminate(); @@ -303,13 +306,13 @@ export class StreamingApiServerService implements OnApplicationShutdown { connection.ping(); } } - }, 1000 * 60); + }, 1000 * 60, { repeated: true }); } @bindThis public async detach(): Promise { if (this.#cleanConnectionsIntervalId) { - clearInterval(this.#cleanConnectionsIntervalId); + this.timeService.stopTimer(this.#cleanConnectionsIntervalId); this.#cleanConnectionsIntervalId = null; } diff --git a/packages/backend/src/server/api/endpoints/admin/ad/list.ts b/packages/backend/src/server/api/endpoints/admin/ad/list.ts index 6406709cda..e3d40d5be4 100644 --- a/packages/backend/src/server/api/endpoints/admin/ad/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/ad/list.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AdsRepository } from '@/models/_.js'; import { QueryService } from '@/core/QueryService.js'; import { DI } from '@/di-symbols.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['admin'], @@ -46,13 +47,14 @@ export default class extends Endpoint { // eslint- private adsRepository: AdsRepository, private queryService: QueryService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const query = this.queryService.makePaginationQuery(this.adsRepository.createQueryBuilder('ad'), ps.sinceId, ps.untilId); if (ps.publishing === true) { - query.andWhere('ad.expiresAt > :now', { now: new Date() }).andWhere('ad.startsAt <= :now', { now: new Date() }); + query.andWhere('ad.expiresAt > :now', { now: this.timeService.date }).andWhere('ad.startsAt <= :now', { now: this.timeService.date }); } else if (ps.publishing === false) { - query.andWhere('ad.expiresAt <= :now', { now: new Date() }).orWhere('ad.startsAt > :now', { now: new Date() }); + query.andWhere('ad.expiresAt <= :now', { now: this.timeService.date }).orWhere('ad.startsAt > :now', { now: this.timeService.date }); } const ads = await query.limit(ps.limit).getMany(); diff --git a/packages/backend/src/server/api/endpoints/admin/announcements/update.ts b/packages/backend/src/server/api/endpoints/admin/announcements/update.ts index 0553ef0426..5a0ebe64ba 100644 --- a/packages/backend/src/server/api/endpoints/admin/announcements/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/announcements/update.ts @@ -9,6 +9,7 @@ import type { AnnouncementsRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { AnnouncementService } from '@/core/AnnouncementService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { TimeService } from '@/global/TimeService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -58,6 +59,7 @@ export default class extends Endpoint { // eslint- private announcementsRepository: AnnouncementsRepository, private announcementService: AnnouncementService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const announcement = await this.announcementsRepository.findOneBy({ id: ps.id }); @@ -66,7 +68,7 @@ export default class extends Endpoint { // eslint- try { await this.announcementService.update(announcement, { - updatedAt: new Date(), + updatedAt: this.timeService.date, title: ps.title, text: ps.text, /* eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- 空の文字列の場合、nullを渡すようにするため */ diff --git a/packages/backend/src/server/api/endpoints/admin/captcha/current.ts b/packages/backend/src/server/api/endpoints/admin/captcha/current.ts index 41192c1926..7cc1bc675a 100644 --- a/packages/backend/src/server/api/endpoints/admin/captcha/current.ts +++ b/packages/backend/src/server/api/endpoints/admin/captcha/current.ts @@ -63,7 +63,11 @@ export const meta = { }, } as const; -export const paramDef = {} as const; +export const paramDef = { + type: 'object', + properties: {}, + required: [], +} as const; @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export diff --git a/packages/backend/src/server/api/endpoints/admin/captcha/save.ts b/packages/backend/src/server/api/endpoints/admin/captcha/save.ts index 98ec278ebe..12b2fdcdf1 100644 --- a/packages/backend/src/server/api/endpoints/admin/captcha/save.ts +++ b/packages/backend/src/server/api/endpoints/admin/captcha/save.ts @@ -5,8 +5,9 @@ import { Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import { captchaErrorCodes, CaptchaService, supportedCaptchaProviders } from '@/core/CaptchaService.js'; +import { CaptchaService, supportedCaptchaProviders } from '@/core/CaptchaService.js'; import { ApiError } from '@/server/api/error.js'; +import { captchaErrorCodes } from '@/misc/captcha-error.js'; export const meta = { tags: ['admin', 'captcha'], diff --git a/packages/backend/src/server/api/endpoints/admin/cw-instance.ts b/packages/backend/src/server/api/endpoints/admin/cw-instance.ts index 5e5d83283e..c7df9ee588 100644 --- a/packages/backend/src/server/api/endpoints/admin/cw-instance.ts +++ b/packages/backend/src/server/api/endpoints/admin/cw-instance.ts @@ -14,8 +14,6 @@ export const meta = { requireCredential: true, requireModerator: true, kind: 'write:admin:cw-instance', - - res: {}, } as const; export const paramDef = { diff --git a/packages/backend/src/server/api/endpoints/admin/cw-note.ts b/packages/backend/src/server/api/endpoints/admin/cw-note.ts index ba2240b8b2..17773c5b81 100644 --- a/packages/backend/src/server/api/endpoints/admin/cw-note.ts +++ b/packages/backend/src/server/api/endpoints/admin/cw-note.ts @@ -19,8 +19,6 @@ export const meta = { requireCredential: true, requireModerator: true, kind: 'write:admin:cw-note', - - res: {}, } as const; export const paramDef = { @@ -99,7 +97,7 @@ export default class extends Endpoint { // eslint- reactionAcceptance: note.reactionAcceptance, visibility: note.visibility, visibleUsers: note.visibleUserIds.length > 0 - ? this.cacheService.getUsers(note.visibleUserIds).then(us => Array.from(us.values())) : null, + ? this.cacheService.findUsersById(note.visibleUserIds).then(us => Array.from(us.values())) : null, channel: note.channel ?? (note.channelId ? this.channelsRepository.findOneByOrFail({ id: note.channelId }) : null), apMentions: undefined, apHashtags: undefined, diff --git a/packages/backend/src/server/api/endpoints/admin/cw-user.ts b/packages/backend/src/server/api/endpoints/admin/cw-user.ts index a7cd2af235..ecb4512fc9 100644 --- a/packages/backend/src/server/api/endpoints/admin/cw-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/cw-user.ts @@ -17,8 +17,6 @@ export const meta = { requireCredential: true, requireModerator: true, kind: 'write:admin:cw-user', - - res: {}, } as const; export const paramDef = { @@ -63,6 +61,8 @@ export default class extends Endpoint { // eslint- userUsername: user.username, userHost: user.host, }); + + return {}; }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/drive/show-file.ts b/packages/backend/src/server/api/endpoints/admin/drive/show-file.ts index a7136d8c8c..361a7d7f95 100644 --- a/packages/backend/src/server/api/endpoints/admin/drive/show-file.ts +++ b/packages/backend/src/server/api/endpoints/admin/drive/show-file.ts @@ -167,6 +167,7 @@ export const paramDef = { fileId: { type: 'string', format: 'misskey:id' }, url: { type: 'string' }, }, + // TODO it chokes on this anyOf: [ { required: ['fileId'] }, { required: ['url'] }, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts index cbf78ada3e..a7d88954d9 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts @@ -66,7 +66,7 @@ export default class extends Endpoint { // eslint- private driveService: DriveService, ) { super(meta, paramDef, async (ps, me) => { - const emoji = await this.emojisRepository.findOneBy({ id: ps.emojiId }); + const emoji = await this.customEmojiService.emojisByIdCache.fetchMaybe(ps.emojiId); if (emoji == null) { throw new ApiError(meta.errors.noSuchEmoji); } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts index 7982c1f0bd..7f4ba083cf 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts @@ -98,6 +98,7 @@ export default class extends Endpoint { // eslint- } if (ps.query) { + // TODO use string inclusion func instead of dynamic query building q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' }) .orderBy('length(emoji.name)', 'ASC'); } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts index 492122422c..c924684e75 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts @@ -102,8 +102,6 @@ export default class extends Endpoint { // eslint- case 'NO_SUCH_EMOJI': throw new ApiError(meta.errors.noSuchEmoji); case 'SAME_NAME_EMOJI_EXISTS': throw new ApiError(meta.errors.sameNameEmojiExists); } - // 網羅性チェック - const mustBeNever: never = error; }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/gen-vapid-keys.ts b/packages/backend/src/server/api/endpoints/admin/gen-vapid-keys.ts index 85e3cd0477..9e4b87f674 100644 --- a/packages/backend/src/server/api/endpoints/admin/gen-vapid-keys.ts +++ b/packages/backend/src/server/api/endpoints/admin/gen-vapid-keys.ts @@ -17,7 +17,11 @@ export const meta = { kind: 'write:admin:meta', } as const; -export const paramDef = {} as const; +export const paramDef = { + type: 'object', + properties: {}, + required: [], +} as const; @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export diff --git a/packages/backend/src/server/api/endpoints/admin/invite/create.ts b/packages/backend/src/server/api/endpoints/admin/invite/create.ts index e52b177e2b..fcdcf84b6e 100644 --- a/packages/backend/src/server/api/endpoints/admin/invite/create.ts +++ b/packages/backend/src/server/api/endpoints/admin/invite/create.ts @@ -5,12 +5,13 @@ import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { RegistrationTicketsRepository } from '@/models/_.js'; +import type { RegistrationTicketsRepository, MiRegistrationTicket } from '@/models/_.js'; import { InviteCodeEntityService } from '@/core/entities/InviteCodeEntityService.js'; import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { generateInviteCode } from '@/misc/generate-invite-code.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; +import { TimeService } from '@/global/TimeService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -57,13 +58,14 @@ export default class extends Endpoint { // eslint- private inviteCodeEntityService: InviteCodeEntityService, private idService: IdService, private moderationLogService: ModerationLogService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { if (ps.expiresAt && isNaN(Date.parse(ps.expiresAt))) { throw new ApiError(meta.errors.invalidDateTime); } - const ticketsPromises = []; + const ticketsPromises: Promise[] = []; for (let i = 0; i < ps.count; i++) { ticketsPromises.push(this.registrationTicketsRepository.insertOne({ @@ -71,7 +73,7 @@ export default class extends Endpoint { // eslint- createdBy: me, createdById: me.id, expiresAt: ps.expiresAt ? new Date(ps.expiresAt) : null, - code: generateInviteCode(), + code: generateInviteCode(this.timeService.now), })); } diff --git a/packages/backend/src/server/api/endpoints/admin/invite/list.ts b/packages/backend/src/server/api/endpoints/admin/invite/list.ts index e33a9a1aec..77f7291a5e 100644 --- a/packages/backend/src/server/api/endpoints/admin/invite/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/invite/list.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { RegistrationTicketsRepository } from '@/models/_.js'; import { InviteCodeEntityService } from '@/core/entities/InviteCodeEntityService.js'; import { DI } from '@/di-symbols.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['admin'], @@ -45,6 +46,7 @@ export default class extends Endpoint { // eslint- private registrationTicketsRepository: RegistrationTicketsRepository, private inviteCodeEntityService: InviteCodeEntityService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const query = this.registrationTicketsRepository.createQueryBuilder('ticket') @@ -54,7 +56,7 @@ export default class extends Endpoint { // eslint- switch (ps.type) { case 'unused': query.andWhere('ticket.usedBy IS NULL'); break; case 'used': query.andWhere('ticket.usedBy IS NOT NULL'); break; - case 'expired': query.andWhere('ticket.expiresAt < :now', { now: new Date() }); break; + case 'expired': query.andWhere('ticket.expiresAt < :now', { now: this.timeService.date }); break; } switch (ps.sort) { diff --git a/packages/backend/src/server/api/endpoints/admin/roles/assign.ts b/packages/backend/src/server/api/endpoints/admin/roles/assign.ts index b6c7953781..9a19be10ba 100644 --- a/packages/backend/src/server/api/endpoints/admin/roles/assign.ts +++ b/packages/backend/src/server/api/endpoints/admin/roles/assign.ts @@ -9,6 +9,7 @@ import type { RolesRepository, UsersRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '@/server/api/error.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['admin', 'role'], @@ -64,6 +65,7 @@ export default class extends Endpoint { // eslint- private rolesRepository: RolesRepository, private roleService: RoleService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const role = await this.rolesRepository.findOneBy({ id: ps.roleId }); @@ -80,7 +82,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchUser); } - if (ps.expiresAt && ps.expiresAt <= Date.now()) { + if (ps.expiresAt && ps.expiresAt <= this.timeService.now) { return; } diff --git a/packages/backend/src/server/api/endpoints/admin/roles/users.ts b/packages/backend/src/server/api/endpoints/admin/roles/users.ts index 0a62845087..db6dd1bc51 100644 --- a/packages/backend/src/server/api/endpoints/admin/roles/users.ts +++ b/packages/backend/src/server/api/endpoints/admin/roles/users.ts @@ -11,6 +11,7 @@ import { QueryService } from '@/core/QueryService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -71,6 +72,7 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private userEntityService: UserEntityService, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const role = await this.rolesRepository.findOneBy({ @@ -86,7 +88,7 @@ export default class extends Endpoint { // eslint- .andWhere(new Brackets(qb => { qb .where('assign.expiresAt IS NULL') - .orWhere('assign.expiresAt > :now', { now: new Date() }); + .orWhere('assign.expiresAt > :now', { now: this.timeService.date }); })) .innerJoinAndSelect('assign.user', 'user'); diff --git a/packages/backend/src/server/api/endpoints/admin/show-users.ts b/packages/backend/src/server/api/endpoints/admin/show-users.ts index 02117d4b6b..adc5c6005d 100644 --- a/packages/backend/src/server/api/endpoints/admin/show-users.ts +++ b/packages/backend/src/server/api/endpoints/admin/show-users.ts @@ -10,6 +10,7 @@ import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['admin'], @@ -61,13 +62,14 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private roleService: RoleService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const query = this.usersRepository.createQueryBuilder('user'); switch (ps.state) { case 'available': query.where('user.isSuspended = FALSE'); break; - case 'alive': query.where('user.updatedAt > :date', { date: new Date(Date.now() - 1000 * 60 * 60 * 24 * 5) }); break; + case 'alive': query.where('user.updatedAt > :date', { date: new Date(this.timeService.now - 1000 * 60 * 60 * 24 * 5) }); break; case 'suspended': query.where('user.isSuspended = TRUE'); break; case 'approved': query.where('user.approved = FALSE'); break; case 'admin': { diff --git a/packages/backend/src/server/api/endpoints/antennas/create.ts b/packages/backend/src/server/api/endpoints/antennas/create.ts index 55d686e390..435a47d8ba 100644 --- a/packages/backend/src/server/api/endpoints/antennas/create.ts +++ b/packages/backend/src/server/api/endpoints/antennas/create.ts @@ -9,8 +9,10 @@ import { IdService } from '@/core/IdService.js'; import type { UserListsRepository, AntennasRepository } from '@/models/_.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { AntennaEntityService } from '@/core/entities/AntennaEntityService.js'; +import { UserListService } from '@/core/UserListService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -97,6 +99,8 @@ export default class extends Endpoint { // eslint- private roleService: RoleService, private idService: IdService, private globalEventService: GlobalEventService, + private readonly timeService: TimeService, + private readonly userListService: UserListService, ) { super(meta, paramDef, async (ps, me) => { if (ps.keywords.flat().every(x => x === '') && ps.excludeKeywords.flat().every(x => x === '')) { @@ -113,17 +117,18 @@ export default class extends Endpoint { // eslint- let userList; if (ps.src === 'list' && ps.userListId) { - userList = await this.userListsRepository.findOneBy({ - id: ps.userListId, - userId: me.id, - }); + userList = await this.userListService.userListsCache.fetchMaybe(ps.userListId); if (userList == null) { throw new ApiError(meta.errors.noSuchUserList); } + + if (!userList.isPublic && userList.userId !== me.id) { + throw new ApiError(meta.errors.noSuchUserList); + } } - const now = new Date(); + const now = this.timeService.date; const antenna = await this.antennasRepository.insertOne({ id: this.idService.gen(now.getTime()), diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index 0aeceda038..683d5b9e91 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -11,6 +11,7 @@ import { QueryService } from '@/core/QueryService.js'; import { DI } from '@/di-symbols.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { trackPromise } from '@/misc/promise-tracker.js'; @@ -77,6 +78,7 @@ export default class extends Endpoint { // eslint- private fanoutTimelineService: FanoutTimelineService, private globalEventService: GlobalEventService, private readonly activeUsersChart: ActiveUsersChart, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -95,7 +97,7 @@ export default class extends Endpoint { // eslint- const needPublishEvent = !antenna.isActive; antenna.isActive = true; - antenna.lastUsedAt = new Date(); + antenna.lastUsedAt = this.timeService.date; trackPromise(this.antennasRepository.update(antenna.id, antenna)); if (needPublishEvent) { diff --git a/packages/backend/src/server/api/endpoints/antennas/update.ts b/packages/backend/src/server/api/endpoints/antennas/update.ts index 3f2513bf75..e9000aa8cd 100644 --- a/packages/backend/src/server/api/endpoints/antennas/update.ts +++ b/packages/backend/src/server/api/endpoints/antennas/update.ts @@ -8,6 +8,8 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AntennasRepository, UserListsRepository } from '@/models/_.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { AntennaEntityService } from '@/core/entities/AntennaEntityService.js'; +import { UserListService } from '@/core/UserListService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../error.js'; @@ -94,6 +96,8 @@ export default class extends Endpoint { // eslint- private antennaEntityService: AntennaEntityService, private globalEventService: GlobalEventService, + private readonly timeService: TimeService, + private readonly userListService: UserListService, ) { super(meta, paramDef, async (ps, me) => { if (ps.keywords && ps.excludeKeywords) { @@ -114,14 +118,15 @@ export default class extends Endpoint { // eslint- let userList; if ((ps.src === 'list' || antenna.src === 'list') && ps.userListId) { - userList = await this.userListsRepository.findOneBy({ - id: ps.userListId, - userId: me.id, - }); + userList = await this.userListService.userListsCache.fetchMaybe(ps.userListId); if (userList == null) { throw new ApiError(meta.errors.noSuchUserList); } + + if (!userList.isPublic && userList.userId !== me.id) { + throw new ApiError(meta.errors.noSuchUserList); + } } await this.antennasRepository.update(antenna.id, { @@ -138,7 +143,7 @@ export default class extends Endpoint { // eslint- withFile: ps.withFile, excludeNotesInSensitiveChannel: ps.excludeNotesInSensitiveChannel, isActive: true, - lastUsedAt: new Date(), + lastUsedAt: this.timeService.date, }); this.globalEventService.publishInternalEvent('antennaUpdated', await this.antennasRepository.findOneByOrFail({ id: antenna.id })); diff --git a/packages/backend/src/server/api/endpoints/auth/accept.ts b/packages/backend/src/server/api/endpoints/auth/accept.ts index 0000ce16ef..1a57f85f7e 100644 --- a/packages/backend/src/server/api/endpoints/auth/accept.ts +++ b/packages/backend/src/server/api/endpoints/auth/accept.ts @@ -8,6 +8,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AuthSessionsRepository, AppsRepository, AccessTokensRepository } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../error.js'; @@ -55,6 +56,7 @@ export default class extends Endpoint { // eslint- private accessTokensRepository: AccessTokensRepository, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { // Fetch token @@ -83,7 +85,7 @@ export default class extends Endpoint { // eslint- sha256.update(accessToken + app.secret); const hash = sha256.digest('hex'); - const now = new Date(); + const now = this.timeService.date; await this.accessTokensRepository.insert({ id: this.idService.gen(now.getTime()), diff --git a/packages/backend/src/server/api/endpoints/bubble-game/ranking.ts b/packages/backend/src/server/api/endpoints/bubble-game/ranking.ts index 1ac3488288..525f6672f9 100644 --- a/packages/backend/src/server/api/endpoints/bubble-game/ranking.ts +++ b/packages/backend/src/server/api/endpoints/bubble-game/ranking.ts @@ -9,6 +9,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { BubbleGameRecordsRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { allowGet: true, @@ -63,12 +64,13 @@ export default class extends Endpoint { // eslint- private bubbleGameRecordsRepository: BubbleGameRecordsRepository, private userEntityService: UserEntityService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const records = await this.bubbleGameRecordsRepository.find({ where: { gameMode: ps.gameMode, - seededAt: MoreThan(new Date(Date.now() - 1000 * 60 * 60 * 24 * 7)), + seededAt: MoreThan(new Date(this.timeService.now - 1000 * 60 * 60 * 24 * 7)), }, order: { score: 'DESC', diff --git a/packages/backend/src/server/api/endpoints/bubble-game/register.ts b/packages/backend/src/server/api/endpoints/bubble-game/register.ts index 0a999e42cd..494c3fcfb9 100644 --- a/packages/backend/src/server/api/endpoints/bubble-game/register.ts +++ b/packages/backend/src/server/api/endpoints/bubble-game/register.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import ms from 'ms'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { BubbleGameRecordsRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../error.js'; @@ -58,10 +59,11 @@ export default class extends Endpoint { // eslint- private bubbleGameRecordsRepository: BubbleGameRecordsRepository, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const seedDate = new Date(parseInt(ps.seed, 10)); - const now = new Date(); + const now = this.timeService.date; // シードが未来なのは通常のプレイではありえないので弾く if (seedDate.getTime() > now.getTime()) { diff --git a/packages/backend/src/server/api/endpoints/channels/create.ts b/packages/backend/src/server/api/endpoints/channels/create.ts index e3a6d2d670..8437f9f060 100644 --- a/packages/backend/src/server/api/endpoints/channels/create.ts +++ b/packages/backend/src/server/api/endpoints/channels/create.ts @@ -6,7 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import ms from 'ms'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { ChannelsRepository, DriveFilesRepository } from '@/models/_.js'; +import type { ChannelsRepository, DriveFilesRepository, MiDriveFile } from '@/models/_.js'; import type { MiChannel } from '@/models/Channel.js'; import { IdService } from '@/core/IdService.js'; import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js'; @@ -68,7 +68,7 @@ export default class extends Endpoint { // eslint- private channelEntityService: ChannelEntityService, ) { super(meta, paramDef, async (ps, me) => { - let banner = null; + let banner: MiDriveFile | null = null; if (ps.bannerId != null) { banner = await this.driveFilesRepository.findOneBy({ id: ps.bannerId, diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index 806b203b51..d10a7b3d39 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -13,7 +13,6 @@ import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; import { MiLocalUser } from '@/models/User.js'; -import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -83,7 +82,6 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private activeUsersChart: ActiveUsersChart, - private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); diff --git a/packages/backend/src/server/api/endpoints/channels/update.ts b/packages/backend/src/server/api/endpoints/channels/update.ts index 7cca688fda..01166f7897 100644 --- a/packages/backend/src/server/api/endpoints/channels/update.ts +++ b/packages/backend/src/server/api/endpoints/channels/update.ts @@ -6,7 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import ms from 'ms'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { DriveFilesRepository, ChannelsRepository } from '@/models/_.js'; +import type { DriveFilesRepository, ChannelsRepository, MiDriveFile } from '@/models/_.js'; import { ChannelEntityService } from '@/core/entities/ChannelEntityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; @@ -99,8 +99,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.accessDenied); } - // eslint:disable-next-line:no-unnecessary-initializer - let banner = undefined; + let banner: MiDriveFile | null = null; if (ps.bannerId != null) { banner = await this.driveFilesRepository.findOneBy({ id: ps.bannerId, diff --git a/packages/backend/src/server/api/endpoints/chat/messages/create-to-room.ts b/packages/backend/src/server/api/endpoints/chat/messages/create-to-room.ts index 0afd3b1ccb..624a1985bd 100644 --- a/packages/backend/src/server/api/endpoints/chat/messages/create-to-room.ts +++ b/packages/backend/src/server/api/endpoints/chat/messages/create-to-room.ts @@ -10,7 +10,7 @@ import { GetterService } from '@/server/api/GetterService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '@/server/api/error.js'; import { ChatService } from '@/core/ChatService.js'; -import type { DriveFilesRepository, MiUser } from '@/models/_.js'; +import type { DriveFilesRepository, MiUser, MiDriveFile } from '@/models/_.js'; import type { Config } from '@/config.js'; export const meta = { @@ -96,7 +96,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchRoom); } - let file = null; + let file: MiDriveFile | null = null; if (ps.fileId != null) { file = await this.driveFilesRepository.findOneBy({ id: ps.fileId, diff --git a/packages/backend/src/server/api/endpoints/chat/messages/create-to-user.ts b/packages/backend/src/server/api/endpoints/chat/messages/create-to-user.ts index 83a83e0d1f..0dbcbbfe77 100644 --- a/packages/backend/src/server/api/endpoints/chat/messages/create-to-user.ts +++ b/packages/backend/src/server/api/endpoints/chat/messages/create-to-user.ts @@ -10,7 +10,7 @@ import { GetterService } from '@/server/api/GetterService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '@/server/api/error.js'; import { ChatService } from '@/core/ChatService.js'; -import type { DriveFilesRepository, MiUser } from '@/models/_.js'; +import type { DriveFilesRepository, MiUser, MiDriveFile } from '@/models/_.js'; import type { Config } from '@/config.js'; export const meta = { @@ -103,7 +103,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.maxLength); } - let file = null; + let file: MiDriveFile | null = null; if (ps.fileId != null) { file = await this.driveFilesRepository.findOneBy({ id: ps.fileId, diff --git a/packages/backend/src/server/api/endpoints/drive/folders/create.ts b/packages/backend/src/server/api/endpoints/drive/folders/create.ts index 08d9d9cdc3..bbe26d7554 100644 --- a/packages/backend/src/server/api/endpoints/drive/folders/create.ts +++ b/packages/backend/src/server/api/endpoints/drive/folders/create.ts @@ -6,7 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import ms from 'ms'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { DriveFoldersRepository } from '@/models/_.js'; +import type { DriveFoldersRepository, MiDriveFolder } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; import { DriveFolderEntityService } from '@/core/entities/DriveFolderEntityService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; @@ -61,7 +61,7 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { // If the parent folder is specified - let parent = null; + let parent: MiDriveFolder | null = null; if (ps.parentId) { // Fetch parent folder parent = await this.driveFoldersRepository.findOneBy({ diff --git a/packages/backend/src/server/api/endpoints/emoji.ts b/packages/backend/src/server/api/endpoints/emoji.ts index 45cc4a10f3..caef5d1528 100644 --- a/packages/backend/src/server/api/endpoints/emoji.ts +++ b/packages/backend/src/server/api/endpoints/emoji.ts @@ -9,6 +9,7 @@ import type { EmojisRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; import { DI } from '@/di-symbols.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; export const meta = { tags: ['meta'], @@ -47,14 +48,10 @@ export default class extends Endpoint { // eslint- private emojisRepository: EmojisRepository, private emojiEntityService: EmojiEntityService, + private readonly customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - const emoji = await this.emojisRepository.findOneOrFail({ - where: { - name: ps.name, - host: IsNull(), - }, - }); + const emoji = await this.customEmojiService.emojisByKeyCache.fetch(ps.name); return this.emojiEntityService.packDetailed(emoji); }); diff --git a/packages/backend/src/server/api/endpoints/emojis.ts b/packages/backend/src/server/api/endpoints/emojis.ts index 4909c948e3..f27db50fe4 100644 --- a/packages/backend/src/server/api/endpoints/emojis.ts +++ b/packages/backend/src/server/api/endpoints/emojis.ts @@ -4,10 +4,13 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import type { EmojisRepository } from '@/models/_.js'; +import { IsNull } from 'typeorm'; +import type { EmojisRepository, MiEmoji } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; import { DI } from '@/di-symbols.js'; +import { CacheManagementService, type ManagedMemorySingleCache } from '@/global/CacheManagementService.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; export const meta = { tags: ['meta'], @@ -32,10 +35,11 @@ export const meta = { }, }, - // 2 calls per second + // Up to 20 calls, then 5 / second limit: { - duration: 1000, - max: 2, + type: 'bucket', + size: 20, + dripRate: 200, }, } as const; @@ -48,21 +52,41 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export + // Short (2 second) cache to handle rapid bursts of fetching the emoji list. + // This just stores the IDs - the actual emojis are cached by CustomEmojiService + private readonly localEmojiIdsCache: ManagedMemorySingleCache; + constructor( @Inject(DI.emojisRepository) private emojisRepository: EmojisRepository, private emojiEntityService: EmojiEntityService, + private readonly customEmojiService: CustomEmojiService, + + cacheManagementService: CacheManagementService, ) { super(meta, paramDef, async (ps, me) => { - const emojis = await this.emojisRepository.createQueryBuilder() - .where('host IS NULL') - .orderBy('LOWER(category)', 'ASC') - .addOrderBy('LOWER(name)', 'ASC') - .getMany(); + // Fetch the latest emoji list + const emojiIds = await this.localEmojiIdsCache.fetch(async () => { + const emojis = await this.emojisRepository.createQueryBuilder('emoji') + .select('emoji.id') + .where({ host: IsNull() }) + .orderBy('LOWER(emoji.category)', 'ASC') + .addOrderBy('LOWER(emoji.name)', 'ASC') + .getMany() as { id: MiEmoji['id'] }[]; + + return emojis.map(e => e.id); + }); + + // Fetch the latest version of each emoji + const emojis = await this.customEmojiService.emojisByIdCache.fetchMany(emojiIds); + + // Pack and return everything return { - emojis: await this.emojiEntityService.packSimpleMany(emojis), + emojis: await this.emojiEntityService.packSimpleMany(emojis.values), }; }); + + this.localEmojiIdsCache = cacheManagementService.createMemorySingleCache('localEmojis', 1000 * 2); } } diff --git a/packages/backend/src/server/api/endpoints/flash/create.ts b/packages/backend/src/server/api/endpoints/flash/create.ts index 588de70b7b..10e9111344 100644 --- a/packages/backend/src/server/api/endpoints/flash/create.ts +++ b/packages/backend/src/server/api/endpoints/flash/create.ts @@ -10,6 +10,7 @@ import { IdService } from '@/core/IdService.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { DI } from '@/di-symbols.js'; import { FlashEntityService } from '@/core/entities/FlashEntityService.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['flash'], @@ -57,12 +58,13 @@ export default class extends Endpoint { // eslint- private flashEntityService: FlashEntityService, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const flash = await this.flashsRepository.insertOne({ id: this.idService.gen(), userId: me.id, - updatedAt: new Date(), + updatedAt: this.timeService.date, title: ps.title, summary: ps.summary, script: ps.script, diff --git a/packages/backend/src/server/api/endpoints/flash/update.ts b/packages/backend/src/server/api/endpoints/flash/update.ts index e378669f0a..58581d0904 100644 --- a/packages/backend/src/server/api/endpoints/flash/update.ts +++ b/packages/backend/src/server/api/endpoints/flash/update.ts @@ -7,6 +7,7 @@ import ms from 'ms'; import { Inject, Injectable } from '@nestjs/common'; import type { FlashsRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../error.js'; @@ -59,6 +60,8 @@ export default class extends Endpoint { // eslint- constructor( @Inject(DI.flashsRepository) private flashsRepository: FlashsRepository, + + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const flash = await this.flashsRepository.findOneBy({ id: ps.flashId }); @@ -70,7 +73,7 @@ export default class extends Endpoint { // eslint- } await this.flashsRepository.update(flash.id, { - updatedAt: new Date(), + updatedAt: this.timeService.date, ...Object.fromEntries( Object.entries(ps).filter( ([key, val]) => (key !== 'flashId') && Object.hasOwn(paramDef.properties, key) diff --git a/packages/backend/src/server/api/endpoints/gallery/featured.ts b/packages/backend/src/server/api/endpoints/gallery/featured.ts index abbfb9b83b..0c62332d07 100644 --- a/packages/backend/src/server/api/endpoints/gallery/featured.ts +++ b/packages/backend/src/server/api/endpoints/gallery/featured.ts @@ -9,6 +9,7 @@ import type { GalleryPostsRepository } from '@/models/_.js'; import { GalleryPostEntityService } from '@/core/entities/GalleryPostEntityService.js'; import { DI } from '@/di-symbols.js'; import { FeaturedService } from '@/core/FeaturedService.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['gallery'], @@ -52,15 +53,16 @@ export default class extends Endpoint { // eslint- private galleryPostEntityService: GalleryPostEntityService, private featuredService: FeaturedService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { let postIds: string[]; - if (this.galleryPostsRankingCacheLastFetchedAt !== 0 && (Date.now() - this.galleryPostsRankingCacheLastFetchedAt < 1000 * 60 * 30)) { + if (this.galleryPostsRankingCacheLastFetchedAt !== 0 && (this.timeService.now - this.galleryPostsRankingCacheLastFetchedAt < 1000 * 60 * 30)) { postIds = this.galleryPostsRankingCache; } else { postIds = await this.featuredService.getGalleryPostsRanking(100); this.galleryPostsRankingCache = postIds; - this.galleryPostsRankingCacheLastFetchedAt = Date.now(); + this.galleryPostsRankingCacheLastFetchedAt = this.timeService.now; } postIds.sort((a, b) => a > b ? -1 : 1); diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts index 08abd7fed5..f280dd8986 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/create.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/create.ts @@ -12,6 +12,7 @@ import type { MiDriveFile } from '@/models/DriveFile.js'; import { IdService } from '@/core/IdService.js'; import { GalleryPostEntityService } from '@/core/entities/GalleryPostEntityService.js'; import { DI } from '@/di-symbols.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['gallery'], @@ -62,6 +63,7 @@ export default class extends Endpoint { // eslint- private galleryPostEntityService: GalleryPostEntityService, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const files = (await Promise.all(ps.fileIds.map(fileId => @@ -77,7 +79,7 @@ export default class extends Endpoint { // eslint- const post = await this.galleryPostsRepository.insertOne(new MiGalleryPost({ id: this.idService.gen(), - updatedAt: new Date(), + updatedAt: this.timeService.date, title: ps.title, description: ps.description, userId: me.id, diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/like.ts b/packages/backend/src/server/api/endpoints/gallery/posts/like.ts index ae8ad6c044..d96f9d8184 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/like.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/like.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { GalleryLikesRepository, GalleryPostsRepository } from '@/models/_.js'; import { FeaturedService, GALLERY_POSTS_RANKING_WINDOW } from '@/core/FeaturedService.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../../error.js'; @@ -66,6 +67,7 @@ export default class extends Endpoint { // eslint- private featuredService: FeaturedService, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const post = await this.galleryPostsRepository.findOneBy({ id: ps.postId }); @@ -97,7 +99,7 @@ export default class extends Endpoint { // eslint- }); // ランキング更新 - if (Date.now() - this.idService.parse(post.id).date.getTime() < GALLERY_POSTS_RANKING_WINDOW) { + if (this.timeService.now - this.idService.parse(post.id).date.getTime() < GALLERY_POSTS_RANKING_WINDOW) { await this.featuredService.updateGalleryPostsRanking(post, 1); } diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts b/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts index be0a5a5584..f5c7d9a61e 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/unlike.ts @@ -8,6 +8,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { GalleryPostsRepository, GalleryLikesRepository } from '@/models/_.js'; import { FeaturedService, GALLERY_POSTS_RANKING_WINDOW } from '@/core/FeaturedService.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../../error.js'; @@ -60,6 +61,7 @@ export default class extends Endpoint { // eslint- private featuredService: FeaturedService, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const post = await this.galleryPostsRepository.findOneBy({ id: ps.postId }); @@ -80,7 +82,7 @@ export default class extends Endpoint { // eslint- await this.galleryLikesRepository.delete(exist.id); // ランキング更新 - if (Date.now() - this.idService.parse(post.id).date.getTime() < GALLERY_POSTS_RANKING_WINDOW) { + if (this.timeService.now - this.idService.parse(post.id).date.getTime() < GALLERY_POSTS_RANKING_WINDOW) { await this.featuredService.updateGalleryPostsRanking(post, -1); } diff --git a/packages/backend/src/server/api/endpoints/gallery/posts/update.ts b/packages/backend/src/server/api/endpoints/gallery/posts/update.ts index d0f9b56863..a7dfe35aca 100644 --- a/packages/backend/src/server/api/endpoints/gallery/posts/update.ts +++ b/packages/backend/src/server/api/endpoints/gallery/posts/update.ts @@ -10,6 +10,7 @@ import type { DriveFilesRepository, GalleryPostsRepository } from '@/models/_.js import type { MiDriveFile } from '@/models/DriveFile.js'; import { GalleryPostEntityService } from '@/core/entities/GalleryPostEntityService.js'; import { DI } from '@/di-symbols.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['gallery'], @@ -60,6 +61,7 @@ export default class extends Endpoint { // eslint- private driveFilesRepository: DriveFilesRepository, private galleryPostEntityService: GalleryPostEntityService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { let files: Array | undefined; @@ -81,7 +83,7 @@ export default class extends Endpoint { // eslint- id: ps.postId, userId: me.id, }, { - updatedAt: new Date(), + updatedAt: this.timeService.date, title: ps.title, description: ps.description, isSensitive: ps.isSensitive, diff --git a/packages/backend/src/server/api/endpoints/get-online-users-count.ts b/packages/backend/src/server/api/endpoints/get-online-users-count.ts index ede777c890..307bdce1cd 100644 --- a/packages/backend/src/server/api/endpoints/get-online-users-count.ts +++ b/packages/backend/src/server/api/endpoints/get-online-users-count.ts @@ -9,6 +9,7 @@ import { USER_ONLINE_THRESHOLD } from '@/const.js'; import type { UsersRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { DI } from '@/di-symbols.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['meta'], @@ -45,10 +46,11 @@ export default class extends Endpoint { // eslint- constructor( @Inject(DI.usersRepository) private usersRepository: UsersRepository, + private readonly timeService: TimeService, ) { super(meta, paramDef, async () => { const count = await this.usersRepository.countBy({ - lastActiveDate: MoreThan(new Date(Date.now() - USER_ONLINE_THRESHOLD)), + lastActiveDate: MoreThan(new Date(this.timeService.now - USER_ONLINE_THRESHOLD)), }); return { diff --git a/packages/backend/src/server/api/endpoints/hashtags/users.ts b/packages/backend/src/server/api/endpoints/hashtags/users.ts index c43e750cf3..1be9f6a553 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/users.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/users.ts @@ -11,6 +11,7 @@ import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { requireCredential: false, @@ -60,6 +61,7 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private readonly roleService: RoleService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { if (!safeForSql(normalizeForSearch(ps.tag))) throw new Error('Injection'); @@ -67,7 +69,7 @@ export default class extends Endpoint { // eslint- .where(':tag <@ user.tags', { tag: [normalizeForSearch(ps.tag)] }) .andWhere('user.isSuspended = FALSE'); - const recent = new Date(Date.now() - (1000 * 60 * 60 * 24 * 5)); + const recent = new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 5)); if (ps.state === 'alive') { query.andWhere('user.updatedAt > :date', { date: recent }); diff --git a/packages/backend/src/server/api/endpoints/i.ts b/packages/backend/src/server/api/endpoints/i.ts index 23e90db356..1c7e8e8bbb 100644 --- a/packages/backend/src/server/api/endpoints/i.ts +++ b/packages/backend/src/server/api/endpoints/i.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import type { UserProfilesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../error.js'; @@ -54,11 +55,12 @@ export default class extends Endpoint { // eslint- private userProfilesRepository: UserProfilesRepository, private userEntityService: UserEntityService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, user, token) => { const isSecure = token == null; - const now = new Date(); + const now = this.timeService.date; const today = `${now.getFullYear()}/${now.getMonth() + 1}/${now.getDate()}`; // 渡ってきている user はキャッシュされていて古い可能性があるので改めて取得 diff --git a/packages/backend/src/server/api/endpoints/i/apps.ts b/packages/backend/src/server/api/endpoints/i/apps.ts index 1563366da2..9fccc10345 100644 --- a/packages/backend/src/server/api/endpoints/i/apps.ts +++ b/packages/backend/src/server/api/endpoints/i/apps.ts @@ -116,7 +116,7 @@ export default class extends Endpoint { // eslint- const tokens = await query.getMany(); - const users = await this.cacheService.getUsers(tokens.flatMap(token => token.granteeIds)); + const users = await this.cacheService.findUsersById(tokens.flatMap(token => token.granteeIds)); const packedUsers = await this.userEntityService.packMany(Array.from(users.values()), me); const packedUserMap = new Map(packedUsers.map(u => [u.id, u])); diff --git a/packages/backend/src/server/api/endpoints/i/import-blocking.ts b/packages/backend/src/server/api/endpoints/i/import-blocking.ts index 78678f33e5..5cc50af1ec 100644 --- a/packages/backend/src/server/api/endpoints/i/import-blocking.ts +++ b/packages/backend/src/server/api/endpoints/i/import-blocking.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueueService } from '@/core/QueueService.js'; import { AccountMoveService } from '@/core/AccountMoveService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { DriveFilesRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../error.js'; @@ -67,6 +68,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, private accountMoveService: AccountMoveService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const file = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); @@ -77,7 +79,7 @@ export default class extends Endpoint { // eslint- const checkMoving = await this.accountMoveService.validateAlsoKnownAs( me, - (old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > Date.now(), + (old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > this.timeService.now, true, ); if (checkMoving ? file.size > 32 * 1024 * 1024 : file.size > 64 * 1024) throw new ApiError(meta.errors.tooBigFile); diff --git a/packages/backend/src/server/api/endpoints/i/import-following.ts b/packages/backend/src/server/api/endpoints/i/import-following.ts index 2d45ff2c8a..bff498a62e 100644 --- a/packages/backend/src/server/api/endpoints/i/import-following.ts +++ b/packages/backend/src/server/api/endpoints/i/import-following.ts @@ -8,6 +8,7 @@ import ms from 'ms'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueueService } from '@/core/QueueService.js'; import { AccountMoveService } from '@/core/AccountMoveService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { DriveFilesRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../error.js'; @@ -69,6 +70,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, private accountMoveService: AccountMoveService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const file = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); @@ -79,7 +81,7 @@ export default class extends Endpoint { // eslint- const checkMoving = await this.accountMoveService.validateAlsoKnownAs( me, - (old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > Date.now(), + (old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > this.timeService.now, true, ); if (checkMoving ? file.size > 32 * 1024 * 1024 : file.size > 64 * 1024) throw new ApiError(meta.errors.tooBigFile); diff --git a/packages/backend/src/server/api/endpoints/i/import-muting.ts b/packages/backend/src/server/api/endpoints/i/import-muting.ts index 1af9cc6263..ed74a84386 100644 --- a/packages/backend/src/server/api/endpoints/i/import-muting.ts +++ b/packages/backend/src/server/api/endpoints/i/import-muting.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueueService } from '@/core/QueueService.js'; import { AccountMoveService } from '@/core/AccountMoveService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { DriveFilesRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../error.js'; @@ -67,6 +68,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, private accountMoveService: AccountMoveService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const file = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); @@ -77,7 +79,7 @@ export default class extends Endpoint { // eslint- const checkMoving = await this.accountMoveService.validateAlsoKnownAs( me, - (old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > Date.now(), + (old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > this.timeService.now, true, ); if (checkMoving ? file.size > 32 * 1024 * 1024 : file.size > 64 * 1024) throw new ApiError(meta.errors.tooBigFile); diff --git a/packages/backend/src/server/api/endpoints/i/import-user-lists.ts b/packages/backend/src/server/api/endpoints/i/import-user-lists.ts index 509483be3f..0d572a86b8 100644 --- a/packages/backend/src/server/api/endpoints/i/import-user-lists.ts +++ b/packages/backend/src/server/api/endpoints/i/import-user-lists.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueueService } from '@/core/QueueService.js'; import { AccountMoveService } from '@/core/AccountMoveService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { DriveFilesRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../error.js'; @@ -67,6 +68,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, private accountMoveService: AccountMoveService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const file = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); @@ -77,7 +79,7 @@ export default class extends Endpoint { // eslint- const checkMoving = await this.accountMoveService.validateAlsoKnownAs( me, - (old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > Date.now(), + (old, src) => !!src.movedAt && src.movedAt.getTime() + 1000 * 60 * 60 * 2 > this.timeService.now, true, ); if (checkMoving ? file.size > 32 * 1024 * 1024 : file.size > 64 * 1024) throw new ApiError(meta.errors.tooBigFile); diff --git a/packages/backend/src/server/api/endpoints/i/notifications-grouped.ts b/packages/backend/src/server/api/endpoints/i/notifications-grouped.ts index 3821b5a20e..4fd962ea4c 100644 --- a/packages/backend/src/server/api/endpoints/i/notifications-grouped.ts +++ b/packages/backend/src/server/api/endpoints/i/notifications-grouped.ts @@ -191,7 +191,7 @@ export default class extends Endpoint { // eslint- // this matches the logic in NotificationService and it's what MkPagination expects if (ps.sinceId && !ps.untilId) groupedNotifications.reverse(); - return await this.notificationEntityService.packGroupedMany(groupedNotifications, me.id); + return await this.notificationEntityService.packGroupedMany(groupedNotifications, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/notifications.ts b/packages/backend/src/server/api/endpoints/i/notifications.ts index f5a48b2f69..c6aeaeb18f 100644 --- a/packages/backend/src/server/api/endpoints/i/notifications.ts +++ b/packages/backend/src/server/api/endpoints/i/notifications.ts @@ -95,7 +95,7 @@ export default class extends Endpoint { // eslint- this.notificationService.readAllNotification(me.id); } - return await this.notificationEntityService.packMany(notifications, me.id); + return await this.notificationEntityService.packMany(notifications, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/i/registry/get.ts b/packages/backend/src/server/api/endpoints/i/registry/get.ts index d284334834..69215e12ce 100644 --- a/packages/backend/src/server/api/endpoints/i/registry/get.ts +++ b/packages/backend/src/server/api/endpoints/i/registry/get.ts @@ -20,8 +20,6 @@ export const meta = { }, }, - res: {}, - // 10 calls per 5 seconds limit: { duration: 1000 * 5, diff --git a/packages/backend/src/server/api/endpoints/invite/create.ts b/packages/backend/src/server/api/endpoints/invite/create.ts index f607a35515..5c3a71f2ed 100644 --- a/packages/backend/src/server/api/endpoints/invite/create.ts +++ b/packages/backend/src/server/api/endpoints/invite/create.ts @@ -10,6 +10,7 @@ import type { RegistrationTicketsRepository } from '@/models/_.js'; import { InviteCodeEntityService } from '@/core/entities/InviteCodeEntityService.js'; import { IdService } from '@/core/IdService.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { generateInviteCode } from '@/misc/generate-invite-code.js'; import { ApiError } from '../../error.js'; @@ -57,13 +58,14 @@ export default class extends Endpoint { // eslint- private inviteCodeEntityService: InviteCodeEntityService, private idService: IdService, private roleService: RoleService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const policies = await this.roleService.getUserPolicies(me.id); if (policies.inviteLimit) { const count = await this.registrationTicketsRepository.countBy({ - id: MoreThan(this.idService.gen(Date.now() - (policies.inviteLimitCycle * 1000 * 60))), + id: MoreThan(this.idService.gen(this.timeService.now - (policies.inviteLimitCycle * 1000 * 60))), createdById: me.id, }); @@ -76,8 +78,8 @@ export default class extends Endpoint { // eslint- id: this.idService.gen(), createdBy: me, createdById: me.id, - expiresAt: policies.inviteExpirationTime ? new Date(Date.now() + (policies.inviteExpirationTime * 1000 * 60)) : null, - code: generateInviteCode(), + expiresAt: policies.inviteExpirationTime ? new Date(this.timeService.now + (policies.inviteExpirationTime * 1000 * 60)) : null, + code: generateInviteCode(this.timeService.now), }); return await this.inviteCodeEntityService.pack(ticket, me); diff --git a/packages/backend/src/server/api/endpoints/invite/limit.ts b/packages/backend/src/server/api/endpoints/invite/limit.ts index 150f4de441..2b1bc8a442 100644 --- a/packages/backend/src/server/api/endpoints/invite/limit.ts +++ b/packages/backend/src/server/api/endpoints/invite/limit.ts @@ -10,6 +10,7 @@ import type { RegistrationTicketsRepository } from '@/models/_.js'; import { RoleService } from '@/core/RoleService.js'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['meta'], @@ -50,12 +51,13 @@ export default class extends Endpoint { // eslint- private roleService: RoleService, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const policies = await this.roleService.getUserPolicies(me.id); const count = policies.inviteLimit ? await this.registrationTicketsRepository.countBy({ - id: MoreThan(this.idService.gen(Date.now() - (policies.inviteLimitCycle * 60 * 1000))), + id: MoreThan(this.idService.gen(this.timeService.now - (policies.inviteLimitCycle * 60 * 1000))), createdById: me.id, }) : null; diff --git a/packages/backend/src/server/api/endpoints/miauth/gen-token.ts b/packages/backend/src/server/api/endpoints/miauth/gen-token.ts index 8644d40538..9cd4cb6d68 100644 --- a/packages/backend/src/server/api/endpoints/miauth/gen-token.ts +++ b/packages/backend/src/server/api/endpoints/miauth/gen-token.ts @@ -11,6 +11,7 @@ import { IdService } from '@/core/IdService.js'; import { NotificationService } from '@/core/NotificationService.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; import { DI } from '@/di-symbols.js'; +import { TimeService } from '@/global/TimeService.js'; import { CacheService } from '@/core/CacheService.js'; export const meta = { @@ -77,12 +78,13 @@ export default class extends Endpoint { // eslint- private idService: IdService, private notificationService: NotificationService, + private readonly timeService: TimeService, private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { // Validate grantees if (ps.grantees && ps.grantees.length > 0) { - const grantees = await this.cacheService.getUsers(ps.grantees); + const grantees = await this.cacheService.findUsersById(ps.grantees); if (grantees.size !== ps.grantees.length) { throw new ApiError(meta.errors.noSuchUser); @@ -98,7 +100,7 @@ export default class extends Endpoint { // eslint- // Generate access token const accessToken = secureRndstr(32); - const now = new Date(); + const now = this.timeService.date; // Insert access token doc await this.accessTokensRepository.insert({ diff --git a/packages/backend/src/server/api/endpoints/mute/create.ts b/packages/backend/src/server/api/endpoints/mute/create.ts index e39c133b43..608f134b00 100644 --- a/packages/backend/src/server/api/endpoints/mute/create.ts +++ b/packages/backend/src/server/api/endpoints/mute/create.ts @@ -10,6 +10,7 @@ import type { MutingsRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { GetterService } from '@/server/api/GetterService.js'; import { UserMutingService } from '@/core/UserMutingService.js'; +import { TimeService } from '@/global/TimeService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -67,6 +68,7 @@ export default class extends Endpoint { // eslint- private getterService: GetterService, private userMutingService: UserMutingService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const muter = me; @@ -94,7 +96,7 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.alreadyMuting); } - if (ps.expiresAt && ps.expiresAt <= Date.now()) { + if (ps.expiresAt && ps.expiresAt <= this.timeService.now) { return; } diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index 4aece5353e..72897922e2 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -18,6 +18,7 @@ import { NoteCreateService } from '@/core/NoteCreateService.js'; import { DI } from '@/di-symbols.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { TimeService } from '@/global/TimeService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -261,6 +262,7 @@ export default class extends Endpoint { // eslint- private noteEntityService: NoteEntityService, private noteCreateService: NoteCreateService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { if (ps.text && ps.text.length > this.config.maxNoteLength) { @@ -331,11 +333,11 @@ export default class extends Endpoint { // eslint- if (ps.poll) { if (typeof ps.poll.expiresAt === 'number') { - if (ps.poll.expiresAt < Date.now()) { + if (ps.poll.expiresAt < this.timeService.now) { throw new ApiError(meta.errors.cannotCreateAlreadyExpiredPoll); } } else if (typeof ps.poll.expiredAfter === 'number') { - ps.poll.expiresAt = Date.now() + ps.poll.expiredAfter; + ps.poll.expiresAt = this.timeService.now + ps.poll.expiredAfter; } } @@ -351,7 +353,7 @@ export default class extends Endpoint { // eslint- // 投稿を作成 try { const note = await this.noteCreateService.create(me, { - createdAt: new Date(), + createdAt: this.timeService.date, files: files, poll: ps.poll ? { choices: ps.poll.choices, diff --git a/packages/backend/src/server/api/endpoints/notes/edit.ts b/packages/backend/src/server/api/endpoints/notes/edit.ts index 276f2672d0..4ffd202eeb 100644 --- a/packages/backend/src/server/api/endpoints/notes/edit.ts +++ b/packages/backend/src/server/api/endpoints/notes/edit.ts @@ -17,6 +17,7 @@ import { NoteEditService } from '@/core/NoteEditService.js'; import { DI } from '@/di-symbols.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { TimeService } from '@/global/TimeService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -311,6 +312,7 @@ export default class extends Endpoint { // eslint- private noteEntityService: NoteEntityService, private noteEditService: NoteEditService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { if (ps.text && ps.text.length > this.config.maxNoteLength) { @@ -381,11 +383,11 @@ export default class extends Endpoint { // eslint- if (ps.poll) { if (typeof ps.poll.expiresAt === 'number') { - if (ps.poll.expiresAt < Date.now()) { + if (ps.poll.expiresAt < this.timeService.now) { throw new ApiError(meta.errors.cannotCreateAlreadyExpiredPoll); } } else if (typeof ps.poll.expiredAfter === 'number') { - ps.poll.expiresAt = Date.now() + ps.poll.expiredAfter; + ps.poll.expiresAt = this.timeService.now + ps.poll.expiredAfter; } } diff --git a/packages/backend/src/server/api/endpoints/notes/featured.ts b/packages/backend/src/server/api/endpoints/notes/featured.ts index ac6dfe8da6..f20046a211 100644 --- a/packages/backend/src/server/api/endpoints/notes/featured.ts +++ b/packages/backend/src/server/api/endpoints/notes/featured.ts @@ -10,10 +10,10 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; import { FeaturedService } from '@/core/FeaturedService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; -import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { ApiError } from '@/server/api/error.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['notes'], @@ -67,11 +67,11 @@ export default class extends Endpoint { // eslint- @Inject(DI.notesRepository) private notesRepository: NotesRepository, - private cacheService: CacheService, private noteEntityService: NoteEntityService, private featuredService: FeaturedService, private queryService: QueryService, private readonly roleService: RoleService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const policies = await this.roleService.getUserPolicies(me ? me.id : null); @@ -83,12 +83,12 @@ export default class extends Endpoint { // eslint- if (ps.channelId) { noteIds = await this.featuredService.getInChannelNotesRanking(ps.channelId, 50); } else { - if (this.globalNotesRankingCacheLastFetchedAt !== 0 && (Date.now() - this.globalNotesRankingCacheLastFetchedAt < 1000 * 60 * 30)) { + if (this.globalNotesRankingCacheLastFetchedAt !== 0 && (this.timeService.now - this.globalNotesRankingCacheLastFetchedAt < 1000 * 60 * 30)) { noteIds = this.globalNotesRankingCache; } else { noteIds = await this.featuredService.getGlobalNotesRanking(100); this.globalNotesRankingCache = noteIds; - this.globalNotesRankingCacheLastFetchedAt = Date.now(); + this.globalNotesRankingCacheLastFetchedAt = this.timeService.now; } } diff --git a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts index 601c5e7a48..816cd771be 100644 --- a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts @@ -12,7 +12,6 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; -import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -68,7 +67,6 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private roleService: RoleService, private activeUsersChart: ActiveUsersChart, - private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const policies = await this.roleService.getUserPolicies(me ? me.id : null); diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 7ad1b0bf6e..c22d3fa5ab 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -12,7 +12,6 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; -import { CacheService } from '@/core/CacheService.js'; import { FanoutTimelineName } from '@/core/FanoutTimelineService.js'; import { QueryService } from '@/core/QueryService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; @@ -90,7 +89,6 @@ export default class extends Endpoint { // eslint- private roleService: RoleService, private activeUsersChart: ActiveUsersChart, private idService: IdService, - private cacheService: CacheService, private queryService: QueryService, private userFollowingService: UserFollowingService, private fanoutTimelineEndpointService: FanoutTimelineEndpointService, diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index 73725f9af2..6b820f9fe9 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -15,7 +15,6 @@ import { IdService } from '@/core/IdService.js'; import { QueryService } from '@/core/QueryService.js'; import { MiLocalUser } from '@/models/User.js'; import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; -import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -84,7 +83,6 @@ export default class extends Endpoint { // eslint- private idService: IdService, private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private queryService: QueryService, - private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); diff --git a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts index 6e5fdaa281..c2a2b16586 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts @@ -12,6 +12,7 @@ import { DI } from '@/di-symbols.js'; import { QueryService } from '@/core/QueryService.js'; import { RoleService } from '@/core/RoleService.js'; import { ApiError } from '@/server/api/error.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['notes'], @@ -77,6 +78,7 @@ export default class extends Endpoint { // eslint- private noteEntityService: NoteEntityService, private readonly queryService: QueryService, private readonly roleService: RoleService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const query = this.pollsRepository.createQueryBuilder('poll') @@ -96,16 +98,16 @@ export default class extends Endpoint { // eslint- if (ps.expired) { query.andWhere('poll.expiresAt IS NOT NULL'); query.andWhere('poll.expiresAt <= :expiresMax', { - expiresMax: new Date(), + expiresMax: this.timeService.date, }); query.andWhere('poll.expiresAt >= :expiresMin', { - expiresMin: new Date(Date.now() - (1000 * 60 * 60 * 24 * 7)), + expiresMin: new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 7)), }); } else { query.andWhere(new Brackets(qb => { qb .where('poll.expiresAt IS NULL') - .orWhere('poll.expiresAt > :now', { now: new Date() }); + .orWhere('poll.expiresAt > :now', { now: this.timeService.date }); })); } diff --git a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts index 0b318304f3..a842347060 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/vote.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/vote.ts @@ -15,6 +15,7 @@ import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { DI } from '@/di-symbols.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; +import { TimeService } from '@/global/TimeService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -101,9 +102,10 @@ export default class extends Endpoint { // eslint- private apRendererService: ApRendererService, private globalEventService: GlobalEventService, private userBlockingService: UserBlockingService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { - const createdAt = new Date(); + const createdAt = this.timeService.date; // Get votee const note = await this.getterService.getNote(ps.noteId).catch(err => { diff --git a/packages/backend/src/server/api/endpoints/notes/schedule/create.ts b/packages/backend/src/server/api/endpoints/notes/schedule/create.ts index 0c2e00ee8d..fcbae5e07c 100644 --- a/packages/backend/src/server/api/endpoints/notes/schedule/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/schedule/create.ts @@ -24,6 +24,7 @@ import { QueueService } from '@/core/QueueService.js'; import { IdService } from '@/core/IdService.js'; import { MiScheduleNoteType } from '@/models/NoteSchedule.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -211,6 +212,7 @@ export default class extends Endpoint { // eslint- private queueService: QueueService, private roleService: RoleService, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const scheduleNoteCount = await this.noteScheduleRepository.countBy({ userId: me.id }); @@ -301,7 +303,7 @@ export default class extends Endpoint { // eslint- } if (ps.poll) { - let scheduleNote_scheduledAt = Date.now(); + let scheduleNote_scheduledAt = this.timeService.now; if (typeof ps.scheduleNote.scheduledAt === 'number') { scheduleNote_scheduledAt = moment.utc(ps.scheduleNote.scheduledAt).local().valueOf(); } @@ -314,7 +316,7 @@ export default class extends Endpoint { // eslint- } } if (typeof ps.scheduleNote.scheduledAt === 'number') { - if (moment.utc(ps.scheduleNote.scheduledAt).local().valueOf() < Date.now()) { + if (moment.utc(ps.scheduleNote.scheduledAt).local().valueOf() < this.timeService.now) { throw new ApiError(meta.errors.cannotCreateAlreadyExpiredSchedule); } } else { @@ -342,7 +344,7 @@ export default class extends Endpoint { // eslint- if (ps.scheduleNote.scheduledAt) { me.token = null; - const noteId = this.idService.gen(new Date().getTime()); + const noteId = this.idService.gen(this.timeService.now); const schedNoteLocalTime = moment.utc(ps.scheduleNote.scheduledAt).local().valueOf(); await this.noteScheduleRepository.insert({ id: noteId, @@ -351,7 +353,7 @@ export default class extends Endpoint { // eslint- scheduledAt: new Date(schedNoteLocalTime), }); - const delay = new Date(schedNoteLocalTime).getTime() - Date.now(); + const delay = new Date(schedNoteLocalTime).getTime() - this.timeService.now; await this.queueService.ScheduleNotePostQueue.add(String(delay), { scheduleNoteId: noteId, }, { diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 27ba7399a7..826d342221 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -12,7 +12,6 @@ import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; -import { CacheService } from '@/core/CacheService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { MiLocalUser } from '@/models/User.js'; import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; @@ -71,7 +70,6 @@ export default class extends Endpoint { // eslint- private noteEntityService: NoteEntityService, private activeUsersChart: ActiveUsersChart, private idService: IdService, - private cacheService: CacheService, private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private userFollowingService: UserFollowingService, private queryService: QueryService, diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index f8c29b60d4..c787c27baa 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -12,11 +12,12 @@ import { GetterService } from '@/server/api/GetterService.js'; import { RoleService } from '@/core/RoleService.js'; import type { MiMeta, MiNote } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; -import { CacheService } from '@/core/CacheService.js'; import { hasText } from '@/models/Note.js'; import { ApiLoggerService } from '@/server/api/ApiLoggerService.js'; import { NoteVisibilityService } from '@/core/NoteVisibilityService.js'; -import { ApiError } from '../../error.js'; +import { CacheManagementService, type ManagedRedisKVCache } from '@/global/CacheManagementService.js'; +import { ApiError } from '@/server/api/error.js'; +import { bindThis } from '@/decorators.js'; export const meta = { tags: ['notes'], @@ -75,6 +76,8 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export + private readonly translationsCache: ManagedRedisKVCache; + constructor( @Inject(DI.meta) private serverSettings: MiMeta, @@ -83,9 +86,10 @@ export default class extends Endpoint { // eslint- private getterService: GetterService, private httpRequestService: HttpRequestService, private roleService: RoleService, - private readonly cacheService: CacheService, private readonly loggerService: ApiLoggerService, private readonly noteVisibilityService: NoteVisibilityService, + + cacheManagementService: CacheManagementService, ) { super(meta, paramDef, async (ps, me) => { const note = await this.getterService.getNote(ps.noteId).catch(err => { @@ -110,7 +114,7 @@ export default class extends Endpoint { // eslint- let targetLang = ps.targetLang; if (targetLang.includes('-')) targetLang = targetLang.split('-')[0]; - let response = await this.cacheService.getCachedTranslation(note, targetLang); + let response = await this.getCachedTranslation(note, targetLang); if (!response) { this.loggerService.logger.debug(`Fetching new translation for note=${note.id} lang=${targetLang}`); response = await this.fetchTranslation(note, targetLang); @@ -118,10 +122,15 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.translationFailed); } - await this.cacheService.setCachedTranslation(note, targetLang, response); + await this.setCachedTranslation(note, targetLang, response); } return response; }); + + this.translationsCache = cacheManagementService.createRedisKVCache('translations', { + lifetime: 1000 * 60 * 60 * 24 * 7, // 1 week, + memoryCacheLifetime: 1000 * 60, // 1 minute + }); } private async fetchTranslation(note: MiNote & { text: string }, targetLang: string) { @@ -219,4 +228,43 @@ export default class extends Endpoint { // eslint- return null; } + + @bindThis + private async getCachedTranslation(note: MiNote, targetLang: string): Promise { + const cacheKey = `${note.id}@${targetLang}`; + + // Use cached translation, if present and up-to-date + const cached = await this.translationsCache.get(cacheKey); + if (cached && cached.u === note.updatedAt?.valueOf()) { + return { + sourceLang: cached.l, + text: cached.t, + }; + } + + // No cache entry :( + return null; + } + + @bindThis + private async setCachedTranslation(note: MiNote, targetLang: string, translation: CachedTranslation): Promise { + const cacheKey = `${note.id}@${targetLang}`; + + await this.translationsCache.set(cacheKey, { + l: translation.sourceLang, + t: translation.text, + u: note.updatedAt?.valueOf(), + }); + } +} + +interface CachedTranslation { + sourceLang: string | undefined; + text: string | undefined; +} + +interface CachedTranslationEntity { + l?: string; + t?: string; + u?: number; } diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index 7fb671a446..517cc822db 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -14,6 +14,8 @@ import { IdService } from '@/core/IdService.js'; import { QueryService } from '@/core/QueryService.js'; import { MiLocalUser } from '@/models/User.js'; import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; +import { UserListService } from '@/core/UserListService.js'; +import { RoleService } from '@/core/RoleService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -87,20 +89,23 @@ export default class extends Endpoint { // eslint- private idService: IdService, private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private queryService: QueryService, + private readonly userListService: UserListService, + private readonly roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); const sinceId = ps.sinceId ?? (ps.sinceDate ? this.idService.gen(ps.sinceDate!) : null); - const list = await this.userListsRepository.findOneBy({ - id: ps.listId, - userId: me.id, - }); + const list = await this.userListService.userListsCache.fetchMaybe(ps.listId); if (list == null) { throw new ApiError(meta.errors.noSuchList); } + if (!list.isPublic && list.userId !== me.id) { + throw new ApiError(meta.errors.noSuchList); + } + if (!this.serverSettings.enableFanoutTimeline) { const timeline = await this.getFromDb(list, { untilId, diff --git a/packages/backend/src/server/api/endpoints/pages/create.ts b/packages/backend/src/server/api/endpoints/pages/create.ts index 1f3ad9281e..907fcb00e0 100644 --- a/packages/backend/src/server/api/endpoints/pages/create.ts +++ b/packages/backend/src/server/api/endpoints/pages/create.ts @@ -5,11 +5,12 @@ import ms from 'ms'; import { Inject, Injectable } from '@nestjs/common'; -import type { DriveFilesRepository, PagesRepository } from '@/models/_.js'; +import type { DriveFilesRepository, PagesRepository, MiDriveFile } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; import { MiPage, pageNameSchema } from '@/models/Page.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { PageEntityService } from '@/core/entities/PageEntityService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../error.js'; @@ -79,9 +80,10 @@ export default class extends Endpoint { // eslint- private pageEntityService: PageEntityService, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { - let eyeCatchingImage = null; + let eyeCatchingImage: MiDriveFile | null = null; if (ps.eyeCatchingImageId != null) { eyeCatchingImage = await this.driveFilesRepository.findOneBy({ id: ps.eyeCatchingImageId, @@ -104,7 +106,7 @@ export default class extends Endpoint { // eslint- const page = await this.pagesRepository.insertOne(new MiPage({ id: this.idService.gen(), - updatedAt: new Date(), + updatedAt: this.timeService.date, title: ps.title, name: ps.name, summary: ps.summary, diff --git a/packages/backend/src/server/api/endpoints/pages/update.ts b/packages/backend/src/server/api/endpoints/pages/update.ts index a6aeb6002e..d149cf52c1 100644 --- a/packages/backend/src/server/api/endpoints/pages/update.ts +++ b/packages/backend/src/server/api/endpoints/pages/update.ts @@ -8,6 +8,7 @@ import { Not } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; import type { PagesRepository, DriveFilesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../error.js'; import { pageNameSchema } from '@/models/Page.js'; @@ -80,6 +81,8 @@ export default class extends Endpoint { // eslint- @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, + + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const page = await this.pagesRepository.findOneBy({ id: ps.pageId }); @@ -114,7 +117,7 @@ export default class extends Endpoint { // eslint- } await this.pagesRepository.update(page.id, { - updatedAt: new Date(), + updatedAt: this.timeService.date, title: ps.title, name: ps.name, summary: ps.summary === undefined ? page.summary : ps.summary, diff --git a/packages/backend/src/server/api/endpoints/ping.ts b/packages/backend/src/server/api/endpoints/ping.ts index ed6f5207a0..4cfdab9e7c 100644 --- a/packages/backend/src/server/api/endpoints/ping.ts +++ b/packages/backend/src/server/api/endpoints/ping.ts @@ -5,6 +5,7 @@ import { Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { requireCredential: false, @@ -38,10 +39,11 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( + private readonly timeService: TimeService, ) { super(meta, paramDef, async () => { return { - pong: Date.now(), + pong: this.timeService.now, }; }); } diff --git a/packages/backend/src/server/api/endpoints/reset-db.ts b/packages/backend/src/server/api/endpoints/reset-db.ts index fe23160bb8..33189717e2 100644 --- a/packages/backend/src/server/api/endpoints/reset-db.ts +++ b/packages/backend/src/server/api/endpoints/reset-db.ts @@ -66,6 +66,8 @@ export default class extends Endpoint { // eslint- logger.info('---- Database reset complete.'); + // Ignore rule - this is just testing code. + // eslint-disable-next-line no-restricted-globals await new Promise(resolve => setTimeout(resolve, 1000)); }); } diff --git a/packages/backend/src/server/api/endpoints/reset-password.ts b/packages/backend/src/server/api/endpoints/reset-password.ts index ba0c60f4ec..813550bbcd 100644 --- a/packages/backend/src/server/api/endpoints/reset-password.ts +++ b/packages/backend/src/server/api/endpoints/reset-password.ts @@ -9,6 +9,7 @@ import type { UserProfilesRepository, PasswordResetRequestsRepository } from '@/ import { Endpoint } from '@/server/api/endpoint-base.js'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['reset password'], @@ -47,6 +48,7 @@ export default class extends Endpoint { // eslint- private userProfilesRepository: UserProfilesRepository, private idService: IdService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const req = await this.passwordResetRequestsRepository.findOneByOrFail({ @@ -54,7 +56,7 @@ export default class extends Endpoint { // eslint- }); // 発行してから30分以上経過していたら無効 - if (Date.now() - this.idService.parse(req.id).date.getTime() > 1000 * 60 * 30) { + if (this.timeService.now - this.idService.parse(req.id).date.getTime() > 1000 * 60 * 30) { throw new Error(); // TODO } diff --git a/packages/backend/src/server/api/endpoints/roles/users.ts b/packages/backend/src/server/api/endpoints/roles/users.ts index a9adac8b1c..38ce3a9be4 100644 --- a/packages/backend/src/server/api/endpoints/roles/users.ts +++ b/packages/backend/src/server/api/endpoints/roles/users.ts @@ -8,6 +8,7 @@ import { Brackets } from 'typeorm'; import type { RoleAssignmentsRepository, RolesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; +import { TimeService } from '@/global/TimeService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApiError } from '../../error.js'; @@ -78,6 +79,7 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private userEntityService: UserEntityService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const role = await this.rolesRepository.findOneBy({ @@ -95,7 +97,7 @@ export default class extends Endpoint { // eslint- .andWhere(new Brackets(qb => { qb .where('assign.expiresAt IS NULL') - .orWhere('assign.expiresAt > :now', { now: new Date() }); + .orWhere('assign.expiresAt > :now', { now: this.timeService.date }); })) .innerJoinAndSelect('assign.user', 'user'); diff --git a/packages/backend/src/server/api/endpoints/users.ts b/packages/backend/src/server/api/endpoints/users.ts index 48942db002..1efab2afca 100644 --- a/packages/backend/src/server/api/endpoints/users.ts +++ b/packages/backend/src/server/api/endpoints/users.ts @@ -11,6 +11,7 @@ import { QueryService } from '@/core/QueryService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { SelectQueryBuilder } from 'typeorm'; export const meta = { @@ -68,6 +69,7 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private queryService: QueryService, private readonly roleService: RoleService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const query = this.usersRepository.createQueryBuilder('user') @@ -75,7 +77,7 @@ export default class extends Endpoint { // eslint- .andWhere('user.isSuspended = FALSE'); switch (ps.state) { - case 'alive': query.andWhere('user.updatedAt > :date', { date: new Date(Date.now() - 1000 * 60 * 60 * 24 * 5) }); break; + case 'alive': query.andWhere('user.updatedAt > :date', { date: new Date(this.timeService.now - 1000 * 60 * 60 * 24 * 5) }); break; } switch (ps.origin) { diff --git a/packages/backend/src/server/api/endpoints/users/lists/create-from-public.ts b/packages/backend/src/server/api/endpoints/users/lists/create-from-public.ts index 94986d22ea..52246033e2 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/create-from-public.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/create-from-public.ts @@ -96,13 +96,9 @@ export default class extends Endpoint { // eslint- private roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { - const listExist = await this.userListsRepository.exists({ - where: { - id: ps.listId, - isPublic: true, - }, - }); + const listExist = await this.userListService.userListsCache.fetchMaybe(ps.listId); if (!listExist) throw new ApiError(meta.errors.noSuchList); + if (!listExist.isPublic && listExist.userId !== me.id) throw new ApiError(meta.errors.noSuchList); const currentCount = await this.userListsRepository.countBy({ userId: me.id, }); @@ -158,7 +154,7 @@ export default class extends Endpoint { // eslint- throw err; } } - return await this.userListEntityService.pack(userList); + return await this.userListEntityService.pack(userList, me.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/lists/create.ts b/packages/backend/src/server/api/endpoints/users/lists/create.ts index c3ea392e89..061253a6a9 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/create.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/create.ts @@ -77,7 +77,7 @@ export default class extends Endpoint { // eslint- name: ps.name, } as MiUserList); - return await this.userListEntityService.pack(userList); + return await this.userListEntityService.pack(userList, me.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/lists/delete.ts b/packages/backend/src/server/api/endpoints/users/lists/delete.ts index 941ce77877..eb5eb6829d 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/delete.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/delete.ts @@ -7,6 +7,8 @@ import { Inject, Injectable } from '@nestjs/common'; import type { UserListsRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { DI } from '@/di-symbols.js'; +import { CacheService } from '@/core/CacheService.js'; +import { UserListService } from '@/core/UserListService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -46,18 +48,29 @@ export default class extends Endpoint { // eslint- constructor( @Inject(DI.userListsRepository) private userListsRepository: UserListsRepository, + + private readonly cacheService: CacheService, + private readonly userListService: UserListService, ) { super(meta, paramDef, async (ps, me) => { - const userList = await this.userListsRepository.findOneBy({ - id: ps.listId, - userId: me.id, - }); + const [userList, listMembership, listFavorites] = await Promise.all([ + this.userListService.userListsCache.fetchMaybe(ps.listId), + this.cacheService.listUserMembershipsCache.fetch(ps.listId), + this.cacheService.listUserFavoritesCache.fetch(ps.listId), + ]); - if (userList == null) { + if (userList == null || userList.userId !== me.id) { throw new ApiError(meta.errors.noSuchList); } - await this.userListsRepository.delete(userList.id); + await Promise.all([ + this.userListsRepository.delete(userList.id), + this.userListService.userListsCache.delete(userList.id), + this.cacheService.listUserFavoritesCache.delete(userList.id), + this.cacheService.listUserMembershipsCache.delete(userList.id), + this.cacheService.userListFavoritesCache.deleteMany(listFavorites), + this.cacheService.userListMembershipsCache.deleteMany(listMembership.keys()), + ]); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/lists/favorite.ts b/packages/backend/src/server/api/endpoints/users/lists/favorite.ts index fa898b0dc7..2f16ad81f1 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/favorite.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/favorite.ts @@ -9,6 +9,8 @@ import type { UserListFavoritesRepository, UserListsRepository } from '@/models/ import { IdService } from '@/core/IdService.js'; import { ApiError } from '@/server/api/error.js'; import { DI } from '@/di-symbols.js'; +import { CacheService } from '@/core/CacheService.js'; +import { UserListService } from '@/core/UserListService.js'; export const meta = { requireCredential: true, @@ -50,28 +52,27 @@ export default class extends Endpoint { @Inject(DI.userListFavoritesRepository) private userListFavoritesRepository: UserListFavoritesRepository, + + private readonly cacheService: CacheService, + private readonly userListService: UserListService, private idService: IdService, ) { super(meta, paramDef, async (ps, me) => { - const userListExist = await this.userListsRepository.exists({ - where: { - id: ps.listId, - isPublic: true, - }, - }); + const [userListExist, myFavorites, listFavorites] = await Promise.all([ + this.userListService.userListsCache.fetchMaybe(ps.listId), + this.cacheService.userListFavoritesCache.fetch(me.id), + this.cacheService.listUserFavoritesCache.fetch(ps.listId), + ]); if (!userListExist) { throw new ApiError(meta.errors.noSuchList); } - const exist = await this.userListFavoritesRepository.exists({ - where: { - userId: me.id, - userListId: ps.listId, - }, - }); + if (!userListExist.isPublic && userListExist.userId !== me.id) { + throw new ApiError(meta.errors.noSuchList); + } - if (exist) { + if (myFavorites.has(ps.listId) || listFavorites.has(me.id)) { throw new ApiError(meta.errors.alreadyFavorited); } @@ -80,6 +81,10 @@ export default class extends Endpoint { userId: me.id, userListId: ps.listId, }); + + // Update caches directly since the Set instances are shared + myFavorites.add(ps.listId); + listFavorites.add(me.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts b/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts index 18373fdf07..559f08b654 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import type { UserListsRepository, UserListFavoritesRepository, UserListMembershipsRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { UserListEntityService } from '@/core/entities/UserListEntityService.js'; +import { UserListService } from '@/core/UserListService.js'; import { DI } from '@/di-symbols.js'; import { QueryService } from '@/core/QueryService.js'; import { ApiError } from '../../../error.js'; @@ -85,21 +86,20 @@ export default class extends Endpoint { private userListEntityService: UserListEntityService, private queryService: QueryService, + private readonly userListService: UserListService, ) { super(meta, paramDef, async (ps, me) => { // Fetch the list - const userList = await this.userListsRepository.findOneBy(!ps.forPublic && me !== null ? { - id: ps.listId, - userId: me.id, - } : { - id: ps.listId, - isPublic: true, - }); + const userList = await this.userListService.userListsCache.fetchMaybe(ps.listId); if (userList == null) { throw new ApiError(meta.errors.noSuchList); } + if (!userList.isPublic && userList.userId !== me?.id) { + throw new ApiError(meta.errors.noSuchList); + } + const query = this.queryService.makePaginationQuery(this.userListMembershipsRepository.createQueryBuilder('membership'), ps.sinceId, ps.untilId) .andWhere('membership.userListId = :userListId', { userListId: userList.id }) .innerJoinAndSelect('membership.user', 'user'); diff --git a/packages/backend/src/server/api/endpoints/users/lists/list.ts b/packages/backend/src/server/api/endpoints/users/lists/list.ts index 7f17863a63..976da9512d 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/list.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/list.ts @@ -88,7 +88,7 @@ export default class extends Endpoint { isPublic: true, }); - return await Promise.all(userLists.map(x => this.userListEntityService.pack(x))); + return await Promise.all(userLists.map(x => this.userListEntityService.pack(x, me?.id))); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/lists/pull.ts b/packages/backend/src/server/api/endpoints/users/lists/pull.ts index 1eb4d4ef42..835bee84c0 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/pull.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/pull.ts @@ -63,12 +63,9 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { // Fetch the list - const userList = await this.userListsRepository.findOneBy({ - id: ps.listId, - userId: me.id, - }); + const userList = await this.userListService.userListsCache.fetchMaybe(ps.listId); - if (userList == null) { + if (userList == null || userList.userId !== me.id) { throw new ApiError(meta.errors.noSuchList); } diff --git a/packages/backend/src/server/api/endpoints/users/lists/push.ts b/packages/backend/src/server/api/endpoints/users/lists/push.ts index c717b3959c..6ddc98aa73 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/push.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/push.ts @@ -9,6 +9,7 @@ import type { UserListsRepository, UserListMembershipsRepository, BlockingsRepos import { Endpoint } from '@/server/api/endpoint-base.js'; import { GetterService } from '@/server/api/GetterService.js'; import { UserListService } from '@/core/UserListService.js'; +import { CacheService } from '@/core/CacheService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../../error.js'; @@ -84,44 +85,31 @@ export default class extends Endpoint { // eslint- private getterService: GetterService, private userListService: UserListService, + private readonly cacheService: CacheService, ) { super(meta, paramDef, async (ps, me) => { - // Fetch the list - const userList = await this.userListsRepository.findOneBy({ - id: ps.listId, - userId: me.id, - }); + const [user, blockings, userList, exist] = await Promise.all([ + this.cacheService.findOptionalUserById(ps.userId), + this.cacheService.userBlockingCache.fetch(ps.userId), + this.userListService.userListsCache.fetchMaybe(ps.listId), + this.cacheService.listUserMembershipsCache.fetch(ps.listId).then(ms => ms.has(ps.userId)), + ]); - if (userList == null) { + if (userList == null || userList.userId !== me.id) { throw new ApiError(meta.errors.noSuchList); } - // Fetch the user - const user = await this.getterService.getUser(ps.userId).catch(err => { - if (err.id === '15348ddd-432d-49c2-8a5a-8069753becff') throw new ApiError(meta.errors.noSuchUser); - throw err; - }); + if (!user) { + throw new ApiError(meta.errors.noSuchUser); + } - // Check blocking if (user.id !== me.id) { - const blockExist = await this.blockingsRepository.exists({ - where: { - blockerId: user.id, - blockeeId: me.id, - }, - }); + const blockExist = blockings.has(me.id); if (blockExist) { throw new ApiError(meta.errors.youHaveBeenBlocked); } } - const exist = await this.userListMembershipsRepository.exists({ - where: { - userListId: userList.id, - userId: user.id, - }, - }); - if (exist) { throw new ApiError(meta.errors.alreadyAdded); } diff --git a/packages/backend/src/server/api/endpoints/users/lists/show.ts b/packages/backend/src/server/api/endpoints/users/lists/show.ts index c7f4128b56..5e10f39be5 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/show.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/show.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import type { UserListsRepository, UserListFavoritesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { UserListEntityService } from '@/core/entities/UserListEntityService.js'; +import { UserListService } from '@/core/UserListService.js'; import { DI } from '@/di-symbols.js'; import { ApiError } from '../../../error.js'; @@ -59,41 +60,21 @@ export default class extends Endpoint { private userListFavoritesRepository: UserListFavoritesRepository, private userListEntityService: UserListEntityService, + private readonly userListService: UserListService, ) { super(meta, paramDef, async (ps, me) => { - const additionalProperties: Partial<{ likedCount: number, isLiked: boolean }> = {}; // Fetch the list - const userList = await this.userListsRepository.findOneBy(!ps.forPublic && me !== null ? { - id: ps.listId, - userId: me.id, - } : { - id: ps.listId, - isPublic: true, - }); + const userList = await this.userListService.userListsCache.fetchMaybe(ps.listId); if (userList == null) { throw new ApiError(meta.errors.noSuchList); } - if (ps.forPublic && userList.isPublic) { - additionalProperties.likedCount = await this.userListFavoritesRepository.countBy({ - userListId: ps.listId, - }); - if (me !== null) { - additionalProperties.isLiked = await this.userListFavoritesRepository.exists({ - where: { - userId: me.id, - userListId: ps.listId, - }, - }); - } else { - additionalProperties.isLiked = false; - } + if (!userList.isPublic && userList.userId !== me?.id) { + throw new ApiError(meta.errors.noSuchList); } - return { - ...await this.userListEntityService.pack(userList), - ...additionalProperties, - }; + + return await this.userListEntityService.pack(userList, me?.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/lists/unfavorite.ts b/packages/backend/src/server/api/endpoints/users/lists/unfavorite.ts index 4d38f7d0a7..2cac1d7557 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/unfavorite.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/unfavorite.ts @@ -8,6 +8,8 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import type { UserListFavoritesRepository, UserListsRepository } from '@/models/_.js'; import { ApiError } from '@/server/api/error.js'; import { DI } from '@/di-symbols.js'; +import { CacheService } from '@/core/CacheService.js'; +import { UserListService } from '@/core/UserListService.js'; export const meta = { requireCredential: true, @@ -49,29 +51,37 @@ export default class extends Endpoint { @Inject(DI.userListFavoritesRepository) private userListFavoritesRepository: UserListFavoritesRepository, + + private readonly cacheService: CacheService, + private readonly userListService: UserListService, ) { super(meta, paramDef, async (ps, me) => { - const userListExist = await this.userListsRepository.exists({ - where: { - id: ps.listId, - isPublic: true, - }, - }); + const [userListExist, myFavorites, listFavorites] = await Promise.all([ + this.userListService.userListsCache.fetchMaybe(ps.listId), + this.cacheService.userListFavoritesCache.fetch(me.id), + this.cacheService.listUserFavoritesCache.fetch(ps.listId), + ]); if (!userListExist) { throw new ApiError(meta.errors.noSuchList); } - const exist = await this.userListFavoritesRepository.findOneBy({ - userListId: ps.listId, - userId: me.id, - }); + if (!userListExist.isPublic && userListExist.userId !== me.id) { + throw new ApiError(meta.errors.noSuchList); + } - if (exist === null) { + if (!myFavorites.has(ps.listId) && !listFavorites.has(me.id)) { throw new ApiError(meta.errors.notFavorited); } - await this.userListFavoritesRepository.delete({ id: exist.id }); + await this.userListFavoritesRepository.delete({ + userId: me.id, + userListId: ps.listId, + }); + + // Update caches directly since the Set instances are shared + myFavorites.delete(ps.listId); + listFavorites.delete(me.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/lists/update-membership.ts b/packages/backend/src/server/api/endpoints/users/lists/update-membership.ts index 0539fadd35..4ff99eaf4e 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/update-membership.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/update-membership.ts @@ -62,12 +62,9 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { // Fetch the list - const userList = await this.userListsRepository.findOneBy({ - id: ps.listId, - userId: me.id, - }); + const userList = await this.userListService.userListsCache.fetchMaybe(ps.listId); - if (userList == null) { + if (userList == null || userList.userId !== me.id) { throw new ApiError(meta.errors.noSuchList); } diff --git a/packages/backend/src/server/api/endpoints/users/lists/update.ts b/packages/backend/src/server/api/endpoints/users/lists/update.ts index ad2f8c02e0..67f4d69325 100644 --- a/packages/backend/src/server/api/endpoints/users/lists/update.ts +++ b/packages/backend/src/server/api/endpoints/users/lists/update.ts @@ -8,6 +8,7 @@ import type { UserListsRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { UserListEntityService } from '@/core/entities/UserListEntityService.js'; import { DI } from '@/di-symbols.js'; +import { UserListService } from '@/core/UserListService.js'; import { ApiError } from '../../../error.js'; export const meta = { @@ -57,23 +58,24 @@ export default class extends Endpoint { // eslint- private userListsRepository: UserListsRepository, private userListEntityService: UserListEntityService, + private readonly userListService: UserListService, ) { super(meta, paramDef, async (ps, me) => { - const userList = await this.userListsRepository.findOneBy({ - id: ps.listId, - userId: me.id, - }); + const userList = await this.userListService.userListsCache.fetchMaybe(ps.listId); - if (userList == null) { + if (userList == null || userList.userId !== me.id) { throw new ApiError(meta.errors.noSuchList); } - await this.userListsRepository.update(userList.id, { - name: ps.name, - isPublic: ps.isPublic, - }); + await Promise.all([ + this.userListsRepository.update(userList.id, { + name: ps.name, + isPublic: ps.isPublic, + }), + this.userListService.userListsCache.delete(userList.id), + ]); - return await this.userListEntityService.pack(userList.id); + return await this.userListEntityService.pack(userList.id, me.id); }); } } diff --git a/packages/backend/src/server/api/endpoints/users/recommendation.ts b/packages/backend/src/server/api/endpoints/users/recommendation.ts index 2585efdc11..c9b698b0e6 100644 --- a/packages/backend/src/server/api/endpoints/users/recommendation.ts +++ b/packages/backend/src/server/api/endpoints/users/recommendation.ts @@ -10,6 +10,7 @@ import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DI } from '@/di-symbols.js'; +import { TimeService } from '@/global/TimeService.js'; export const meta = { tags: ['users'], @@ -62,13 +63,14 @@ export default class extends Endpoint { // eslint- private userEntityService: UserEntityService, private queryService: QueryService, + private readonly timeService: TimeService, ) { super(meta, paramDef, async (ps, me) => { const query = this.usersRepository.createQueryBuilder('user') .where('user.isLocked = FALSE') .andWhere('user.isExplorable = TRUE') .andWhere('user.host IS NULL') - .andWhere('user.updatedAt >= :date', { date: new Date(Date.now() - ms('7days')) }) + .andWhere('user.updatedAt >= :date', { date: new Date(this.timeService.now - ms('7days')) }) .andWhere('user.id != :meId', { meId: me.id }) .orderBy('user.followersCount', 'DESC'); diff --git a/packages/backend/src/server/api/endpoints/users/report-abuse.ts b/packages/backend/src/server/api/endpoints/users/report-abuse.ts index fc2b57c4a5..a60d624726 100644 --- a/packages/backend/src/server/api/endpoints/users/report-abuse.ts +++ b/packages/backend/src/server/api/endpoints/users/report-abuse.ts @@ -65,10 +65,7 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { // Lookup user - const targetUser = await this.cacheService.findOptionalUserById(ps.userId); - if (!targetUser) { - throw new ApiError(meta.errors.noSuchUser); - } + const targetUser = await this.cacheService.findUserById(ps.userId); if (targetUser.id === me.id) { throw new ApiError(meta.errors.cannotReportYourself); diff --git a/packages/backend/src/server/api/error.ts b/packages/backend/src/server/api/error.ts index 2f8322a568..576f13aeac 100644 --- a/packages/backend/src/server/api/error.ts +++ b/packages/backend/src/server/api/error.ts @@ -6,6 +6,9 @@ type E = { message: string, code: string, id: string, kind?: 'client' | 'server' | 'permission', httpStatusCode?: number }; export class ApiError extends Error { + // Fix the error name in stack traces - https://stackoverflow.com/a/71573071 + override name = this.constructor.name; + public message: string; public code: string; public id: string; diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 072dacf708..1d2a1db625 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { createReadStream } from 'node:fs'; +import { Readable } from 'node:stream'; import { Injectable } from '@nestjs/common'; import { bindThis } from '@/decorators.js'; import { getErrorData, getErrorException, getErrorStatus, MastodonLogger } from '@/server/api/mastodon/MastodonLogger.js'; @@ -118,7 +120,10 @@ export class MastodonApiServerService { } const client = this.clientService.getClient(_request); - const data = await client.uploadMedia(multipartData); + const data = await client.uploadMedia({ + ...multipartData, + stream: Readable.toWeb(createReadStream(multipartData.filepath)), + }); const response = convertAttachment(data.data as Entity.Attachment); return reply.send(response); @@ -131,7 +136,10 @@ export class MastodonApiServerService { } const client = this.clientService.getClient(_request); - const data = await client.uploadMedia(multipartData, _request.body); + const data = await client.uploadMedia({ + ...multipartData, + stream: Readable.toWeb(createReadStream(multipartData.filepath)), + }, _request.body); const response = convertAttachment(data.data as Entity.Attachment); return reply.send(response); diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts index 42a1836e5c..6b0283bf55 100644 --- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts +++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts @@ -5,9 +5,9 @@ import { Inject, Injectable } from '@nestjs/common'; import { Entity, MastodonEntity, MisskeyEntity } from 'megalodon'; -import mfm from 'mfm-js'; -import { MastodonNotificationType } from 'megalodon/lib/src/mastodon/notification.js'; -import { NotificationType } from 'megalodon/lib/src/notification.js'; +import * as mfm from 'mfm-js'; +import { MastodonNotificationType } from 'megalodon/built/lib/mastodon/notification.js'; +import { NotificationType } from 'megalodon/built/lib/notification.js'; import { DI } from '@/di-symbols.js'; import { MfmService } from '@/core/MfmService.js'; import type { Config } from '@/config.js'; @@ -287,7 +287,7 @@ export class MastodonConverters { this.getUser(p) .then(u => this.encode(u, mentionedRemoteUsers)) .catch(() => null))) - .then((p: Entity.Mention[]) => p.filter(m => m)); + .then((p: (Entity.Mention | null)[]) => p.filter(m => m != null)); const tags = note.tags.map(tag => { return { @@ -345,7 +345,7 @@ export class MastodonConverters { sensitive: status.sensitive || !!cw, spoiler_text: cw, visibility: status.visibility, - media_attachments: status.media_attachments.map((a: Entity.Account) => convertAttachment(a)), + media_attachments: status.media_attachments.map((a: Entity.Attachment) => convertAttachment(a)), mentions: mentions, tags: tags, card: null, //FIXME diff --git a/packages/backend/src/server/api/mastodon/MastodonLogger.ts b/packages/backend/src/server/api/mastodon/MastodonLogger.ts index 5ea69ed151..dda9fcfd6b 100644 --- a/packages/backend/src/server/api/mastodon/MastodonLogger.ts +++ b/packages/backend/src/server/api/mastodon/MastodonLogger.ts @@ -67,7 +67,7 @@ export function getErrorException(error: unknown): Error | null { } // This is the inner exception, basically - if (error.cause && !isAxiosError(error.cause)) { + if (error.cause instanceof Error && !isAxiosError(error.cause)) { if (!error.cause.stack) { error.cause.stack = error.stack; } @@ -150,7 +150,7 @@ function unpackAxiosError(error: unknown): unknown { return undefined; } - if (error.cause && !isAxiosError(error.cause)) { + if (error.cause instanceof Error && !isAxiosError(error.cause)) { if (!error.cause.stack) { error.cause.stack = error.stack; } diff --git a/packages/backend/src/server/api/mastodon/endpoints/instance.ts b/packages/backend/src/server/api/mastodon/endpoints/instance.ts index f502b5e191..109b02f2d4 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/instance.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/instance.ts @@ -41,8 +41,7 @@ export class ApiInstanceMastodon { const response: MastodonEntity.Instance = { uri: this.config.host, title: this.meta.name || 'Sharkey', - shortDescription: this.meta.description || 'This is a vanilla Sharkey Instance. It doesn\'t seem to have a description.', - description: this.meta.about || 'This is a vanilla Sharkey Instance.', + description: this.meta.description || this.meta.about || 'This is a vanilla Sharkey Instance.', email: instance.email || '', version: `3.0.0 (compatible; Sharkey ${this.config.version}; like Akkoma)`, urls: instance.urls, diff --git a/packages/backend/src/server/api/mastodon/endpoints/search.ts b/packages/backend/src/server/api/mastodon/endpoints/search.ts index c43d6cfc9a..0d1df8c06b 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/search.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/search.ts @@ -112,7 +112,7 @@ export class ApiSearchMastodon { { method: 'POST', headers: { - ...request.headers, + ...request.headers as Record, 'Accept': 'application/json', 'Content-Type': 'application/json', }, @@ -135,7 +135,7 @@ export class ApiSearchMastodon { { method: 'POST', headers: { - ...request.headers, + ...request.headers as Record, 'Accept': 'application/json', 'Content-Type': 'application/json', }, diff --git a/packages/backend/src/server/api/mastodon/endpoints/status.ts b/packages/backend/src/server/api/mastodon/endpoints/status.ts index 7a058a0ed9..f5942a5267 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/status.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/status.ts @@ -10,7 +10,6 @@ import { parseTimelineArgs, TimelineArgs, toBoolean, toInt } from '@/server/api/ import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import { MastodonDataService } from '@/server/api/mastodon/MastodonDataService.js'; import { getNoteSummary } from '@/misc/get-note-summary.js'; -import type { Packed } from '@/misc/json-schema.js'; import { isPureRenote } from '@/misc/is-renote.js'; import { convertAttachment, convertPoll, MastodonConverters } from '../MastodonConverters.js'; import type { Entity } from 'megalodon'; @@ -46,7 +45,34 @@ export class ApiStatusMastodon { // Fixup - Discord ignores CWs and renders the entire post. if (response.sensitive && _request.headers['user-agent']?.match(/\bDiscordbot\//)) { - response.content = getNoteSummary(data.data satisfies Packed<'Note'>); + // TODO move this mastoConverters? + response.content = getNoteSummary({ + ...data.data, + user: { + ...data.data.account, + emojis: {}, + noindex: data.data.account.noindex ?? false, + }, + visibility: data.data.visibility === 'direct' + ? 'specified' + : data.data.visibility === 'private' + ? 'followers' + : data.data.visibility === 'unlisted' + ? 'home' + : data.data.visibility, + mentions: data.data.mentions.map(m => m.id), + tags: data.data.tags.map(t => t.name), + poll: data.data.poll && { + ...data.data.poll, + choices: data.data.poll.options.map(o => ({ + ...o, + text: o.title, + votes: o.votes_count ?? 0, + isVoted: o.votes_count != null, + })), + }, + emojis: {}, + }); response.media_attachments = []; response.in_reply_to_id = null; response.in_reply_to_account_id = null; @@ -182,7 +208,7 @@ export class ApiStatusMastodon { if (body.in_reply_to_id && removed === '/unreact') { const id = body.in_reply_to_id; const post = await client.getStatus(id); - const react = post.data.emoji_reactions.filter((e: Entity.Emoji) => e.me)[0].name; + const react = post.data.emoji_reactions.filter((e: Entity.Reaction) => e.me)[0].name; const data = await client.deleteEmojiReaction(id, react); return reply.send(data.data); } diff --git a/packages/backend/src/server/api/openapi/gen-spec.ts b/packages/backend/src/server/api/openapi/gen-spec.ts index 85d1fd0bce..49ff1b15f6 100644 --- a/packages/backend/src/server/api/openapi/gen-spec.ts +++ b/packages/backend/src/server/api/openapi/gen-spec.ts @@ -10,7 +10,7 @@ import { getSchemas, convertSchemaToOpenApiSchema } from './schemas.js'; export function genOpenapiSpec(config: Config, includeSelfRef = false) { const spec = { - openapi: '3.1.0', + openapi: '3.1', info: { version: config.version, diff --git a/packages/backend/src/server/api/openapi/schemas.ts b/packages/backend/src/server/api/openapi/schemas.ts index c80dda8d96..af0bbc22c9 100644 --- a/packages/backend/src/server/api/openapi/schemas.ts +++ b/packages/backend/src/server/api/openapi/schemas.ts @@ -13,6 +13,22 @@ export function convertSchemaToOpenApiSchema(schema: Schema, type: 'param' | 're const { optional, nullable, ref, selfRef, ..._res }: any = schema; const res = deepClone(_res); + // "required" must be an array of strings, or undefined. + if (res.required !== undefined) { + // Single-item must be wrapped in array + if (!Array.isArray(res.required)) { + res.required = [res.required]; + } + + // Array must contain only strings + res.required = res.required.filter((required: unknown) => typeof(required) === 'string'); + + // Can't be an empty array + if (res.required.length === 0) { + delete res.required; + } + } + if (schema.type === 'object' && schema.properties) { if (type === 'res') { const required = Object.entries(schema.properties).filter(([k, v]) => !v.optional).map(([k]) => k); @@ -32,20 +48,26 @@ export function convertSchemaToOpenApiSchema(schema: Schema, type: 'param' | 're } for (const o of ['anyOf', 'oneOf', 'allOf'] as const) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - if (o in schema) res[o] = schema[o]!.map(schema => convertSchemaToOpenApiSchema(schema, type, includeSelfRef)); + if (type === 'param') { + // params cannot contain oneOf/allOf/anyOf/etc. + // https://stackoverflow.com/a/29708580 + delete res[o]; + } else { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + if (o in schema) res[o] = schema[o]!.map(schema => convertSchemaToOpenApiSchema(schema, type, includeSelfRef)); + } } if (type === 'res' && schema.ref && (!schema.selfRef || includeSelfRef)) { const $ref = `#/components/schemas/${schema.ref}`; + // https://stackoverflow.com/a/23737104 if (schema.nullable || schema.optional) { - res.allOf = [{ $ref }]; + res.oneOf = [{ $ref }, { type: 'null' }]; } else { res.$ref = $ref; } - } - - if (schema.nullable) { + delete res.type; + } else if (schema.nullable) { if (Array.isArray(schema.type) && !schema.type.includes('null')) { res.type.push('null'); } else if (typeof schema.type === 'string') { diff --git a/packages/backend/src/server/api/stream/Connection.ts b/packages/backend/src/server/api/stream/Connection.ts index 5bce5eda41..d7f33c53a9 100644 --- a/packages/backend/src/server/api/stream/Connection.ts +++ b/packages/backend/src/server/api/stream/Connection.ts @@ -16,6 +16,7 @@ import { ChannelFollowingService } from '@/core/ChannelFollowingService.js'; import { isJsonObject } from '@/misc/json-value.js'; import type { JsonObject, JsonValue } from '@/misc/json-value.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { TimeService, type TimerHandle } from '@/global/TimeService.js'; import type Logger from '@/logger.js'; import { QueryService } from '@/core/QueryService.js'; import type { ChannelsService } from './ChannelsService.js'; @@ -48,7 +49,7 @@ export default class Connection { public myRecentReactions: Map = new Map(); public myRecentRenotes: Set = new Set(); public myRecentFavorites: Set = new Set(); - private fetchIntervalId: NodeJS.Timeout | null = null; + private fetchIntervalId: TimerHandle | null = null; private closingConnection = false; private logger: Logger; @@ -61,6 +62,8 @@ export default class Connection { private notificationService: NotificationService, public readonly cacheService: CacheService, private channelFollowingService: ChannelFollowingService, + private readonly timeService: TimeService, + loggerService: LoggerService, user: MiUser | null | undefined, @@ -80,7 +83,7 @@ export default class Connection { const [userProfile, following, followingChannels, userIdsWhoMeMuting, userIdsWhoBlockingMe, userIdsWhoMeMutingRenotes, threadMutings, noteMutings, myRecentReactions, myRecentFavorites, myRecentRenotes] = await Promise.all([ this.cacheService.userProfileCache.fetch(this.user.id), this.cacheService.userFollowingsCache.fetch(this.user.id), - this.channelFollowingService.userFollowingChannelsCache.fetch(this.user.id), + this.cacheService.userFollowingChannelsCache.fetch(this.user.id), this.cacheService.userMutingsCache.fetch(this.user.id), this.cacheService.userBlockedCache.fetch(this.user.id), this.cacheService.renoteMutingsCache.fetch(this.user.id), @@ -126,7 +129,7 @@ export default class Connection { await this.fetch(); if (!this.fetchIntervalId) { - this.fetchIntervalId = setInterval(this.fetch, 1000 * 10); + this.fetchIntervalId = this.timeService.startTimer(this.fetch, 1000 * 10, { repeated: true }); } } } @@ -376,7 +379,7 @@ export default class Connection { */ @bindThis public dispose() { - if (this.fetchIntervalId) clearInterval(this.fetchIntervalId); + if (this.fetchIntervalId) this.timeService.stopTimer(this.fetchIntervalId); for (const c of this.channels.values()) { if (c.dispose) c.dispose(); } diff --git a/packages/backend/src/server/api/stream/channels/user-list.ts b/packages/backend/src/server/api/stream/channels/user-list.ts index fe5ec38195..b9e2f92161 100644 --- a/packages/backend/src/server/api/stream/channels/user-list.ts +++ b/packages/backend/src/server/api/stream/channels/user-list.ts @@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import type { MiUserListMembership, UserListMembershipsRepository, UserListsRepository } from '@/models/_.js'; import type { Packed } from '@/misc/json-schema.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; +import { UserListService } from '@/core/UserListService.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { isPackedPureRenote } from '@/misc/is-renote.js'; @@ -19,14 +20,13 @@ class UserListChannel extends Channel { public static requireCredential = true as const; public static kind = 'read:account'; private listId: string; - private membershipsMap: Record | undefined> = {}; - private listUsersClock: NodeJS.Timeout; private withFiles: boolean; private withRenotes: boolean; constructor( private userListsRepository: UserListsRepository, private userListMembershipsRepository: UserListMembershipsRepository, + private readonly userListService: UserListService, noteEntityService: NoteEntityService, id: string, @@ -45,39 +45,14 @@ class UserListChannel extends Channel { this.withRenotes = !!(params.withRenotes ?? true); // Check existence and owner - const listExist = await this.userListsRepository.exists({ - where: { - id: this.listId, - userId: this.user!.id, - }, - }); + const listExist = await this.userListService.userListsCache.fetchMaybe(this.listId); if (!listExist) return; + if (!listExist.isPublic && listExist.userId !== this.user?.id) return; // Subscribe stream this.subscriber?.on(`userListStream:${this.listId}`, this.send); this.subscriber?.on('notesStream', this.onNote); - - this.updateListUsers(); - this.listUsersClock = setInterval(this.updateListUsers, 5000); - } - - @bindThis - private async updateListUsers() { - const memberships = await this.userListMembershipsRepository.find({ - where: { - userListId: this.listId, - }, - select: ['userId'], - }); - - const membershipsMap: Record | undefined> = {}; - for (const membership of memberships) { - membershipsMap[membership.userId] = { - withReplies: membership.withReplies, - }; - } - this.membershipsMap = membershipsMap; } @bindThis @@ -87,9 +62,10 @@ class UserListChannel extends Channel { if (this.withFiles && (note.fileIds == null || note.fileIds.length === 0)) return; - if (!Object.hasOwn(this.membershipsMap, note.userId)) return; + const memberships = await this.cacheService.listUserMembershipsCache.fetch(this.listId); + if (!memberships.has(note.userId)) return; - const { accessible, silence } = await this.checkNoteVisibility(note, { includeReplies: true }); + const { accessible, silence } = await this.checkNoteVisibility(note, { includeReplies: true, listContext: this.listId }); if (!accessible || silence) return; if (!this.withRenotes && isPackedPureRenote(note)) return; @@ -102,8 +78,6 @@ class UserListChannel extends Channel { // Unsubscribe events this.subscriber?.off(`userListStream:${this.listId}`, this.send); this.subscriber?.off('notesStream', this.onNote); - - clearInterval(this.listUsersClock); } } @@ -121,6 +95,7 @@ export class UserListChannelService implements MiChannelService { private userListMembershipsRepository: UserListMembershipsRepository, private noteEntityService: NoteEntityService, + private readonly userListService: UserListService, ) { } @@ -129,6 +104,7 @@ export class UserListChannelService implements MiChannelService { return new UserListChannel( this.userListsRepository, this.userListMembershipsRepository, + this.userListService, this.noteEntityService, id, connection, diff --git a/packages/backend/src/server/oauth/OAuth2ProviderService.ts b/packages/backend/src/server/oauth/OAuth2ProviderService.ts index 01ee451297..5fa31abbc4 100644 --- a/packages/backend/src/server/oauth/OAuth2ProviderService.ts +++ b/packages/backend/src/server/oauth/OAuth2ProviderService.ts @@ -11,6 +11,7 @@ import { DI } from '@/di-symbols.js'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; import { getErrorData } from '@/server/api/mastodon/MastodonLogger.js'; import { ServerUtilityService } from '@/server/ServerUtilityService.js'; +import { TimeService } from '@/global/TimeService.js'; import type { FastifyInstance } from 'fastify'; const kinds = [ @@ -56,6 +57,7 @@ export class OAuth2ProviderService { private readonly mastodonClientService: MastodonClientService, private readonly serverUtilityService: ServerUtilityService, + private readonly timeService: TimeService, ) { } // https://datatracker.ietf.org/doc/html/rfc8414.html @@ -118,7 +120,7 @@ export class OAuth2ProviderService { access_token: uuid(), token_type: 'Bearer', scope: 'read', - created_at: Math.floor(new Date().getTime() / 1000), + created_at: Math.floor(this.timeService.now / 1000), }; return reply.send(ret); } @@ -138,7 +140,7 @@ export class OAuth2ProviderService { access_token: atData.accessToken, token_type: 'Bearer', scope: atData.scope || body.scope || 'read write follow push', - created_at: atData.createdAt || Math.floor(new Date().getTime() / 1000), + created_at: atData.createdAt || Math.floor(this.timeService.now / 1000), }; return reply.send(ret); } catch (e: unknown) { diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index 654d477628..c6fd8f4fb1 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -20,18 +20,6 @@ import type { Config } from '@/config.js'; import { getNoteSummary } from '@/misc/get-note-summary.js'; import { DI } from '@/di-symbols.js'; import * as Acct from '@/misc/acct.js'; -import type { - DbQueue, - DeliverQueue, - EndedPollNotificationQueue, - InboxQueue, - ObjectStorageQueue, - RelationshipQueue, - SystemQueue, - UserWebhookDeliverQueue, - SystemWebhookDeliverQueue, - ScheduleNotePostQueue, -} from '@/core/QueueModule.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { PageEntityService } from '@/core/entities/PageEntityService.js'; @@ -57,6 +45,7 @@ import { handleRequestRedirectToOmitSearch } from '@/misc/fastify-hook-handlers. import { bindThis } from '@/decorators.js'; import { FlashEntityService } from '@/core/entities/FlashEntityService.js'; import { RoleService } from '@/core/RoleService.js'; +import { TimeService } from '@/global/TimeService.js'; import { ReversiGameEntityService } from '@/core/entities/ReversiGameEntityService.js'; import { AnnouncementEntityService } from '@/core/entities/AnnouncementEntityService.js'; import { FeedService } from './FeedService.js'; @@ -130,17 +119,7 @@ export class ClientServerService { private feedService: FeedService, private roleService: RoleService, private clientLoggerService: ClientLoggerService, - - @Inject('queue:system') public systemQueue: SystemQueue, - @Inject('queue:endedPollNotification') public endedPollNotificationQueue: EndedPollNotificationQueue, - @Inject('queue:deliver') public deliverQueue: DeliverQueue, - @Inject('queue:inbox') public inboxQueue: InboxQueue, - @Inject('queue:db') public dbQueue: DbQueue, - @Inject('queue:relationship') public relationshipQueue: RelationshipQueue, - @Inject('queue:objectStorage') public objectStorageQueue: ObjectStorageQueue, - @Inject('queue:userWebhookDeliver') public userWebhookDeliverQueue: UserWebhookDeliverQueue, - @Inject('queue:systemWebhookDeliver') public systemWebhookDeliverQueue: SystemWebhookDeliverQueue, - @Inject('queue:scheduleNotePost') public scheduleNotePostQueue: ScheduleNotePostQueue, + private readonly timeService: TimeService, ) { //this.createServer = this.createServer.bind(this); } @@ -216,7 +195,7 @@ export class ClientServerService { instanceUrl: this.config.url, randomMOTD: this.config.customMOTD ? this.config.customMOTD[Math.floor(Math.random() * this.config.customMOTD.length)] : undefined, metaJson: htmlSafeJsonStringify(await this.metaEntityService.packDetailed(meta)), - now: Date.now(), + now: this.timeService.now, }; } diff --git a/packages/backend/src/server/web/FeedService.ts b/packages/backend/src/server/web/FeedService.ts index a622ae7e34..e53cfc7c0c 100644 --- a/packages/backend/src/server/web/FeedService.ts +++ b/packages/backend/src/server/web/FeedService.ts @@ -6,6 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { In, IsNull } from 'typeorm'; import { Feed } from 'feed'; +import { parse as mfmParse } from 'mfm-js'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository, NotesRepository, UserProfilesRepository } from '@/models/_.js'; import type { Config } from '@/config.js'; @@ -15,7 +16,7 @@ import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.j import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; import { MfmService } from "@/core/MfmService.js"; -import { parse as mfmParse } from 'mfm-js'; +import { TimeService } from '@/global/TimeService.js'; @Injectable() export class FeedService { @@ -36,6 +37,7 @@ export class FeedService { private driveFileEntityService: DriveFileEntityService, private idService: IdService, private mfmService: MfmService, + private readonly timeService: TimeService, ) { } @@ -107,7 +109,7 @@ export class FeedService { private shouldHideNote(reference: number | null, createdAt: Date): boolean { if ((reference !== null) && ( - (reference <= 0 && (Date.now() - createdAt.getTime() > 0 - (reference * 1000))) + (reference <= 0 && (this.timeService.now - createdAt.getTime() > 0 - (reference * 1000))) || (reference > 0 && (createdAt.getTime() < reference * 1000)) ) ) { diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 438aa5905d..7a98b25f4e 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -6,7 +6,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { summaly } from '@misskey-dev/summaly'; import { SummalyResult } from '@misskey-dev/summaly/built/summary.js'; -import * as Redis from 'ioredis'; import { IsNull, Not } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; @@ -16,7 +15,6 @@ import { query } from '@/misc/prelude/url.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import { MiMeta } from '@/models/Meta.js'; -import { RedisKVCache } from '@/misc/cache.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import type { MiAccessToken, NotesRepository } from '@/models/_.js'; @@ -27,6 +25,7 @@ import { SystemAccountService } from '@/core/SystemAccountService.js'; import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; import { AuthenticateService, AuthenticationError } from '@/server/api/AuthenticateService.js'; import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; +import { CacheManagementService, type ManagedRedisKVCache } from '@/global/CacheManagementService.js'; import { BucketRateLimit, Keyed, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import type { MiLocalUser } from '@/models/User.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; @@ -69,15 +68,12 @@ const previewLimit: Keyed = { @Injectable() export class UrlPreviewService { private logger: Logger; - private previewCache: RedisKVCache; + private previewCache: ManagedRedisKVCache; constructor( @Inject(DI.config) private config: Config, - @Inject(DI.redis) - private readonly redisClient: Redis.Redis, - @Inject(DI.meta) private readonly meta: MiMeta, @@ -95,9 +91,11 @@ export class UrlPreviewService { private readonly apNoteService: ApNoteService, private readonly authenticateService: AuthenticateService, private readonly rateLimiterService: SkRateLimiterService, + + cacheManagementService: CacheManagementService, ) { this.logger = this.loggerService.getLogger('url-preview'); - this.previewCache = new RedisKVCache(this.redisClient, 'summaly', { + this.previewCache = cacheManagementService.createRedisKVCache('summaly', { lifetime: 1000 * 60 * 60 * 24, // 1d memoryCacheLifetime: 1000 * 60 * 10, // 10m fetcher: () => { throw new Error('the UrlPreview cache should never fetch'); }, diff --git a/packages/backend/src/server/web/bios.js b/packages/backend/src/server/web/bios.js index 5dbb26f9e3..e0f0907b67 100644 --- a/packages/backend/src/server/web/bios.js +++ b/packages/backend/src/server/web/bios.js @@ -8,8 +8,8 @@ window.onload = async () => { const content = document.getElementById('content'); - document.getElementById('ls').addEventListener('click', () => { - content.innerHTML = ''; + document.getElementById('ls')?.addEventListener('click', () => { + if (content) content.innerHTML = ''; const lsEditor = document.createElement('div'); lsEditor.id = 'lsEditor'; @@ -31,7 +31,7 @@ window.onload = async () => { lsEditor.appendChild(adder); for (let i = 0; i < localStorage.length; i++) { - const k = localStorage.key(i); + const k = /** @type {string} */ (localStorage.key(i)); const record = document.createElement('div'); record.classList.add('record'); const header = document.createElement('header'); @@ -57,6 +57,6 @@ window.onload = async () => { lsEditor.appendChild(record); } - content.appendChild(lsEditor); + content?.appendChild(lsEditor); }); }; diff --git a/packages/backend/src/server/web/boot.embed.js b/packages/backend/src/server/web/boot.embed.js index 4f27e2fb30..5aa326dc71 100644 --- a/packages/backend/src/server/web/boot.embed.js +++ b/packages/backend/src/server/web/boot.embed.js @@ -18,7 +18,7 @@ let forceError = localStorage.getItem('forceError'); if (forceError != null) { - renderError('FORCED_ERROR', 'This error is forced by having forceError in local storage.'); + renderError('FORCED_ERROR'); return; } @@ -43,6 +43,7 @@ //#region Detect language & fetch translations if (!localStorage.getItem('locale')) { const supportedLangs = LANGS; + /** @type {string | null | undefined} */ let lang = localStorage.getItem('lang'); if (lang == null || !supportedLangs.includes(lang)) { if (supportedLangs.includes(navigator.language)) { @@ -92,12 +93,20 @@ } //#endregion + /** + * @param {string} styleText + * @returns {Promise} + */ async function addStyle(styleText) { let css = document.createElement('style'); css.appendChild(document.createTextNode(styleText)); document.head.appendChild(css); } + /** + * @param {string} code + * @returns {Promise} + */ async function renderError(code) { // Cannot set property 'innerHTML' of null を回避 if (document.readyState === 'loading') { diff --git a/packages/backend/src/server/web/boot.js b/packages/backend/src/server/web/boot.js index c10f6a54ae..81ceec1f12 100644 --- a/packages/backend/src/server/web/boot.js +++ b/packages/backend/src/server/web/boot.js @@ -34,6 +34,7 @@ //#region Detect language & fetch translations if (!localStorage.getItem('locale')) { const supportedLangs = LANGS; + /** @type {string | null | undefined} */ let lang = localStorage.getItem('lang'); if (lang == null || !supportedLangs.includes(lang)) { if (supportedLangs.includes(navigator.language)) { @@ -154,12 +155,21 @@ document.head.appendChild(style); } + /** + * @param {string} styleText + * @returns {Promise} + */ async function addStyle(styleText) { let css = document.createElement('style'); css.appendChild(document.createTextNode(styleText)); document.head.appendChild(css); } + /** + * @param {string} code + * @param {any} [details] + * @returns {Promise} + */ async function renderError(code, details) { // Cannot set property 'innerHTML' of null を回避 if (document.readyState === 'loading') { @@ -233,7 +243,7 @@ ERROR CODE: ${code} ${details.toString()} ${JSON.stringify(details)}`; - errorsElement.appendChild(detailsElement); + errorsElement?.appendChild(detailsElement); addStyle(` * { font-family: BIZ UDGothic, Roboto, HelveticaNeue, Arial, sans-serif; diff --git a/packages/backend/src/server/web/cli.js b/packages/backend/src/server/web/cli.js index 30ee77f4d9..5322ac7ab5 100644 --- a/packages/backend/src/server/web/cli.js +++ b/packages/backend/src/server/web/cli.js @@ -6,9 +6,15 @@ 'use strict'; window.onload = async () => { - const account = JSON.parse(localStorage.getItem('account')); - const i = account.token; + const accountRaw = localStorage.getItem('account'); + const account = accountRaw ? JSON.parse(accountRaw) : null; + const i = account?.token; + /** + * @param {string} endpoint + * @param {Record} data + * @returns {Promise} + */ const api = (endpoint, data = {}) => { const promise = new Promise((resolve, reject) => { // Append a credential @@ -17,19 +23,19 @@ window.onload = async () => { // Send request fetch(endpoint.indexOf('://') > -1 ? endpoint : `/api/${endpoint}`, { headers: { - 'Content-Type': 'application/json' + 'Content-Type': 'application/json', }, method: 'POST', body: JSON.stringify(data), credentials: 'omit', - cache: 'no-cache' + cache: 'no-cache', }).then(async (res) => { const body = res.status === 204 ? null : await res.json(); if (res.status === 200) { resolve(body); } else if (res.status === 204) { - resolve(); + resolve({}); } else { reject(body.error); } @@ -39,9 +45,9 @@ window.onload = async () => { return promise; }; - document.getElementById('submit').addEventListener('click', () => { + document.getElementById('submit')?.addEventListener('click', () => { api('notes/create', { - text: document.getElementById('text').value + text: (/** @type {HTMLInputElement} */(document.getElementById('text'))).value }).then(() => { location.reload(); }); @@ -49,6 +55,7 @@ window.onload = async () => { api('notes/timeline').then(notes => { const tl = document.getElementById('tl'); + if (!tl) return; for (const note of notes) { const el = document.createElement('div'); const name = document.createElement('header'); diff --git a/packages/backend/src/server/web/error.js b/packages/backend/src/server/web/error.js index 4c6ae730b3..5ea45b5aab 100644 --- a/packages/backend/src/server/web/error.js +++ b/packages/backend/src/server/web/error.js @@ -29,6 +29,7 @@ el.textContent = reload; } + /** @type {NodeListOf} */ const i18nEls = document.querySelectorAll('[data-i18n]'); for (const el of i18nEls) { const key = el.dataset.i18n; diff --git a/packages/backend/src/server/web/global.d.ts b/packages/backend/src/server/web/global.d.ts new file mode 100644 index 0000000000..db55d05b2d --- /dev/null +++ b/packages/backend/src/server/web/global.d.ts @@ -0,0 +1,3 @@ +declare const CLIENT_ENTRY: string; +declare const LANGS_VERSION: string; +declare const LANGS: string[]; diff --git a/packages/backend/test-federation/eslint.config.js b/packages/backend/test-federation/eslint.config.js index e3bcf4c0fe..0df1c7a23a 100644 --- a/packages/backend/test-federation/eslint.config.js +++ b/packages/backend/test-federation/eslint.config.js @@ -1,15 +1,25 @@ import globals from 'globals'; import tsParser from '@typescript-eslint/parser'; -import sharedConfig from '../../shared/eslint.config.js'; +import pluginMisskey from '@misskey-dev/eslint-plugin'; export default [ - ...sharedConfig, { - files: ['**/*.ts', '**/*.tsx'], + ignores: [ + "**/built/", + '*.js', + ], + }, + { languageOptions: { globals: { ...globals.node, }, + }, + }, + { + ...pluginMisskey.configs['typescript'], + files: ['daemon.ts', 'test/**/*.ts'], + languageOptions: { parserOptions: { parser: tsParser, project: ['./tsconfig.json'], @@ -17,5 +27,18 @@ export default [ tsconfigRootDir: import.meta.dirname, }, }, + rules: { + 'no-restricted-syntax': [ + 'error', + { + "selector": "CallExpression[callee.property.name='delete'][arguments.length=1] > ObjectExpression[properties.length=0]", + "message": "repository.delete({}) will produce a runtime error. Use repository.deleteAll() instead." + }, + { + "selector": "CallExpression[callee.property.name='update'][arguments.length>=1] > ObjectExpression[properties.length=0]", + "message": "repository.update({}, {...}) will produce a runtime error. Use repository.updateAll({...}) instead." + }, + ], + } }, ]; diff --git a/packages/backend/test-federation/test/utils.ts b/packages/backend/test-federation/test/utils.ts index b155103efa..4470fbe9eb 100644 --- a/packages/backend/test-federation/test/utils.ts +++ b/packages/backend/test-federation/test/utils.ts @@ -187,7 +187,7 @@ export async function uploadFile( path = '../../test/resources/192.jpg', ): Promise { const filename = path.split('/').pop() ?? 'untitled'; - const blob = new Blob([await readFile(join(__dirname, path))]); + const blob = new Blob([await readFile(join(__dirname, path)) as Buffer]); const body = new FormData(); body.append('i', user.i); @@ -196,7 +196,7 @@ export async function uploadFile( body.append('name', filename); return await fetch(`https://${host}/api/drive/files/create`, { method: 'POST', body }) - .then(async res => await res.json()); + .then(async res => await res.json() as Misskey.entities.DriveFile); } export async function addCustomEmoji( diff --git a/packages/backend/test-federation/tsconfig.json b/packages/backend/test-federation/tsconfig.json index 16b333f877..b20788831c 100644 --- a/packages/backend/test-federation/tsconfig.json +++ b/packages/backend/test-federation/tsconfig.json @@ -1,111 +1,19 @@ { + "extends": "../../shared/tsconfig.node.jsonc", "compilerOptions": { - /* Visit https://aka.ms/tsconfig to read more about this file */ + // Checking + "types": ["jest", "node"], + "verbatimModuleSyntax": false, + "noImplicitOverride": false, + "noImplicitAny": false, + "strictFunctionTypes": false, + "strictPropertyInitialization": false, + "paths": { + "@/*": ["../src/*"] + }, - /* Projects */ - "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ - // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ - // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ - // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ - // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ - // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ - - /* Language and Environment */ - "target": "ESNext", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ - // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ - // "jsx": "preserve", /* Specify what JSX code is generated. */ - // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ - // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ - // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ - // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ - // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ - // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ - // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ - // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ - // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ - - /* Modules */ - "module": "NodeNext", /* Specify what module code is generated. */ - // "rootDir": "./", /* Specify the root folder within your source files. */ - // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ - // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ - // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ - // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ - // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ - // "types": [], /* Specify type package names to be included without being referenced in a source file. */ - // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ - // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ - // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ - // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ - // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ - // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ - // "noUncheckedSideEffectImports": true, /* Check side effect imports. */ - // "resolveJsonModule": true, /* Enable importing .json files. */ - // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ - // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ - - /* JavaScript Support */ - // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ - // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ - // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ - - /* Emit */ - // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ - // "declarationMap": true, /* Create sourcemaps for d.ts files. */ - // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ - // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ - // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ - // "noEmit": true, /* Disable emitting files from a compilation. */ - // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ - "outDir": "./built", /* Specify an output folder for all emitted files. */ - // "removeComments": true, /* Disable emitting comments. */ - // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ - // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ - // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ - // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ - // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ - // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ - // "newLine": "crlf", /* Set the newline character for emitting files. */ - // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ - // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ - // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ - // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ - // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ - - /* Interop Constraints */ - // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ - // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ - // "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */ - // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ - "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ - // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ - "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ - - /* Type Checking */ - "strict": true, /* Enable all strict type-checking options. */ - // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ - // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ - // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ - // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ - // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ - // "strictBuiltinIteratorReturn": true, /* Built-in iterators are instantiated with a 'TReturn' type of 'undefined' instead of 'any'. */ - // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ - // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ - // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ - // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ - // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ - // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ - // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ - // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ - // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ - // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ - // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ - // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ - // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ - - /* Completeness */ - // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ - "skipLibCheck": true /* Skip type checking all .d.ts files. */ + // Output + "outDir": "./built" }, "include": [ "daemon.ts", diff --git a/packages/backend/test-server/.swcrc b/packages/backend/test-server/.swcrc index eeac7eabc6..6cf275985e 100644 --- a/packages/backend/test-server/.swcrc +++ b/packages/backend/test-server/.swcrc @@ -17,7 +17,7 @@ "paths": { "@/*": ["*"] }, - "target": "es2022" + "target": "ESNext" }, "minify": false } diff --git a/packages/backend/test-server/entry.ts b/packages/backend/test-server/entry.ts index 1e0de094cb..eb3e4c9b13 100644 --- a/packages/backend/test-server/entry.ts +++ b/packages/backend/test-server/entry.ts @@ -4,11 +4,11 @@ import Fastify from 'fastify'; import { NestFactory } from '@nestjs/core'; import { MainModule } from '@/MainModule.js'; import { ServerService } from '@/server/ServerService.js'; -import { loadConfig } from '@/config.js'; +import type { Config } from '@/config.js'; import { NestLogger } from '@/NestLogger.js'; +import { DI } from '@/di-symbols.js'; import { INestApplicationContext } from '@nestjs/common'; -const config = loadConfig(); const originEnv = JSON.stringify(process.env); process.env.NODE_ENV = 'test'; @@ -20,18 +20,21 @@ let serverService: ServerService; * テスト用のサーバインスタンスを起動する */ async function launch() { - await killTestServer(); - - console.log('starting application...'); - app = await NestFactory.createApplicationContext(MainModule, { logger: new NestLogger(), }); app.enableShutdownHooks(); + + const config = app.get(DI.config); + + await killTestServer(config); + + console.log('starting application...'); + serverService = app.get(ServerService); await serverService.launch(); - await startControllerEndpoints(); + await startControllerEndpoints(config); // ジョブキューは必要な時にテストコード側で起動する // ジョブキューが動くとテスト結果の確認に支障が出ることがあるので意図的に動かさないでいる @@ -42,7 +45,7 @@ async function launch() { /** * 既に重複したポートで待ち受けしているサーバがある場合はkillする */ -async function killTestServer() { +async function killTestServer(config: Config) { // try { const pid = await portToPid(config.port); @@ -56,9 +59,10 @@ async function killTestServer() { /** * 別プロセスに切り離してしまったが故に出来なくなった環境変数の書き換え等を実現するためのエンドポイントを作る - * @param port + * @param config */ -async function startControllerEndpoints(port = config.port + 1000) { +async function startControllerEndpoints(config: Config) { + const port = config.port + 1000; const fastify = Fastify(); fastify.post<{ Body: { key?: string, value?: string } }>('/env', async (req, res) => { @@ -80,7 +84,7 @@ async function startControllerEndpoints(port = config.port + 1000) { await serverService.dispose(); await app.close(); - await killTestServer(); + await killTestServer(config); console.log('starting application...'); diff --git a/packages/backend/test-server/eslint.config.js b/packages/backend/test-server/eslint.config.js index b9c16d469f..cac45eda4c 100644 --- a/packages/backend/test-server/eslint.config.js +++ b/packages/backend/test-server/eslint.config.js @@ -38,6 +38,17 @@ export default [ name: '__filename', message: 'Not in ESModule. Use `import.meta.url` instead.', }], + 'no-restricted-syntax': [ + 'error', + { + "selector": "CallExpression[callee.property.name='delete'][arguments.length=1] > ObjectExpression[properties.length=0]", + "message": "repository.delete({}) will produce a runtime error. Use repository.deleteAll() instead." + }, + { + "selector": "CallExpression[callee.property.name='update'][arguments.length>=1] > ObjectExpression[properties.length=0]", + "message": "repository.update({}, {...}) will produce a runtime error. Use repository.updateAll({...}) instead." + }, + ], }, }, ]; diff --git a/packages/backend/test-server/tsconfig.json b/packages/backend/test-server/tsconfig.json index cb394ecccd..50a1108937 100644 --- a/packages/backend/test-server/tsconfig.json +++ b/packages/backend/test-server/tsconfig.json @@ -1,53 +1,23 @@ { + "extends": "../tsconfig.backend.json", "compilerOptions": { - "allowJs": true, - "noEmitOnError": true, - "noImplicitAny": true, - "noImplicitReturns": true, - "noUnusedParameters": false, - "noUnusedLocals": false, - "noFallthroughCasesInSwitch": true, - "declaration": false, - "sourceMap": true, - "target": "ES2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "allowSyntheticDefaultImports": true, - "removeComments": false, - "noLib": false, - "strict": true, - "strictNullChecks": true, - "strictPropertyInitialization": false, - "skipLibCheck": true, - "experimentalDecorators": true, - "emitDecoratorMetadata": true, - "resolveJsonModule": true, - "isolatedModules": true, - "incremental": true, - "rootDir": "../src", - "baseUrl": "./", + "rootDir": "../", "paths": { "@/*": ["../src/*"] }, - "outDir": "../built-test", - "types": [ - "node" - ], - "typeRoots": [ - "../src/@types", - "../node_modules/@types", - "../node_modules" - ], - "lib": [ - "esnext" - ] + "outDir": "../built-test" }, - "compileOnSave": false, "include": [ "./**/*.ts", "../src/**/*.ts" ], "exclude": [ + "**/node_modules", + "**/built/", + "**/*.test.ts", + "./test/**/*", + "./test-federation/**/*", + "./test-server/**/*", "../src/**/*.test.ts" ] } diff --git a/packages/backend/test/e2e/2fa.ts b/packages/backend/test/e2e/2fa.ts index 289359a2ce..1cce37c2fb 100644 --- a/packages/backend/test/e2e/2fa.ts +++ b/packages/backend/test/e2e/2fa.ts @@ -20,11 +20,15 @@ import type { RegistrationResponseJSON, } from '@simplewebauthn/types'; import type * as misskey from 'misskey-js'; +import { NativeTimeService } from '@/global/TimeService.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import { EnvService } from '@/global/EnvService.js'; describe('2要素認証', () => { let alice: misskey.entities.SignupResponse; - const config = loadConfig(); + const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService()); + const config = loadConfig(loggerService); const password = 'test'; const username = 'alice'; diff --git a/packages/backend/test/e2e/move.ts b/packages/backend/test/e2e/move.ts index fd798bdb25..e1252cf9fe 100644 --- a/packages/backend/test/e2e/move.ts +++ b/packages/backend/test/e2e/move.ts @@ -15,6 +15,9 @@ import { secureRndstr } from '@/misc/secure-rndstr.js'; import { jobQueue } from '@/boot/common.js'; import { api, castAsError, initTestDb, signup, successfulApiCall, uploadFile } from '../utils.js'; import type * as misskey from 'misskey-js'; +import { LoggerService } from '@/core/LoggerService.js'; +import { NativeTimeService } from '@/global/TimeService.js'; +import { EnvService } from '@/global/EnvService.js'; describe('Account Move', () => { let jq: INestApplicationContext; @@ -33,7 +36,8 @@ describe('Account Move', () => { beforeAll(async () => { jq = await jobQueue(); - const config = loadConfig(); + const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService()); + const config = loadConfig(loggerService); url = new URL(config.url); const connection = await initTestDb(false); root = await signup({ username: 'root' }); diff --git a/packages/backend/test/e2e/oauth.ts b/packages/backend/test/e2e/oauth.ts index 97830bc987..810b5ca434 100644 --- a/packages/backend/test/e2e/oauth.ts +++ b/packages/backend/test/e2e/oauth.ts @@ -153,13 +153,14 @@ async function assertDirectError(response: Response, status: number, error: stri } describe('OAuth', () => { - test('fake pass', () => { - assert.ok(true, 'fake pass'); - }); + test('fake pass', () => { + assert.ok(true, 'fake pass'); + }); }); // these tests won't pass until we integrate Misskey's OAuth code with ours -if (false) describe('OAuth', () => { +/* +describe('OAuth', () => { let fastify: FastifyInstance; let alice: misskey.entities.SignupResponse; @@ -1025,3 +1026,4 @@ if (false) describe('OAuth', () => { }); }); }); + */ diff --git a/packages/backend/test/e2e/reversi-game.ts b/packages/backend/test/e2e/reversi-game.ts index 788255beac..919ab70913 100644 --- a/packages/backend/test/e2e/reversi-game.ts +++ b/packages/backend/test/e2e/reversi-game.ts @@ -6,7 +6,6 @@ process.env.NODE_ENV = 'test'; import * as assert from 'assert'; -import { ReversiMatchResponse } from 'misskey-js/entities.js'; import { api, signup } from '../utils.js'; import type * as misskey from 'misskey-js'; @@ -26,7 +25,7 @@ describe('ReversiGame', () => { const response2 = await api('reversi/match', { userId: alice.id }, bob); assert.strictEqual(response2.status, 200); assert.notStrictEqual(response2.body, null); - const body = response2.body as ReversiMatchResponse; + const body = response2.body as NonNullable; assert.strictEqual(body.user1.id, alice.id); assert.strictEqual(body.user2.id, bob.id); }); diff --git a/packages/backend/test/e2e/timelines.ts b/packages/backend/test/e2e/timelines.ts index 9f97c7314c..8462bd65db 100644 --- a/packages/backend/test/e2e/timelines.ts +++ b/packages/backend/test/e2e/timelines.ts @@ -12,6 +12,9 @@ import { Redis } from 'ioredis'; import { api, post, randomString, sendEnvUpdateRequest, signup, uploadUrl, withNotesCount, initTestDb } from '../utils.js'; import { loadConfig } from '@/config.js'; import { MiInstance } from '@/models/Instance.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import { NativeTimeService } from '@/global/TimeService.js'; +import { EnvService } from '@/global/EnvService.js'; async function genHost() { const hostname = randomString() + '.example.com'; @@ -33,7 +36,12 @@ let redisForTimelines: Redis; describe('Timelines', () => { beforeAll(async () => { - redisForTimelines = new Redis(loadConfig().redisForTimelines); + const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService()); + redisForTimelines = new Redis(loadConfig(loggerService).redisForTimelines); + }); + + afterAll(() => { + redisForTimelines.disconnect(); }); describe('Home TL', () => { diff --git a/packages/backend/test/eslint.config.js b/packages/backend/test/eslint.config.js index a0f43babad..e80ff0cbe9 100644 --- a/packages/backend/test/eslint.config.js +++ b/packages/backend/test/eslint.config.js @@ -18,5 +18,25 @@ export default [ tsconfigRootDir: import.meta.dirname, }, }, + rules: { + 'no-restricted-syntax': [ + 'error', + { + "selector": "CallExpression[callee.property.name='delete'][arguments.length=1] > ObjectExpression[properties.length=0]", + "message": "repository.deleteAll() will produce a runtime error. Use repository.deleteAll() instead." + }, + { + "selector": "CallExpression[callee.property.name='update'][arguments.length>=1] > ObjectExpression[properties.length=0]", + "message": "repository.update({}, {...}) will produce a runtime error. Use repository.updateAll({...}) instead." + }, + ], + } + }, + { + ignores: [ + "**/built/", + '*.*', + "**/jest.setup.*" + ], }, ]; diff --git a/packages/backend/test/jest.setup.ts b/packages/backend/test/jest.setup.e2e.mjs similarity index 100% rename from packages/backend/test/jest.setup.ts rename to packages/backend/test/jest.setup.e2e.mjs diff --git a/packages/backend/test/jest.setup.unit.mjs b/packages/backend/test/jest.setup.unit.mjs new file mode 100644 index 0000000000..da3f715d8a --- /dev/null +++ b/packages/backend/test/jest.setup.unit.mjs @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +// Need to reference the built file +import { prepEnv } from '../built/boot/prepEnv.js'; + +/** + * Applies the Sharkey environment details into the test environment. + * https://jestjs.io/docs/configuration#globalsetup-string + */ +export default function setup() { + // Make sure tests run in the Sharkey environment. + prepEnv(); +} diff --git a/packages/backend/test/misc/FakeCacheManagementService.ts b/packages/backend/test/misc/FakeCacheManagementService.ts new file mode 100644 index 0000000000..bd6c8aa8ef --- /dev/null +++ b/packages/backend/test/misc/FakeCacheManagementService.ts @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import { GodOfTimeService } from './GodOfTimeService.js'; +import { MockInternalEventService } from './MockInternalEventService.js'; +import { MockRedis } from './MockRedis.js'; +import type * as Redis from 'ioredis'; +import type { QuantumKVOpts } from '@/misc/QuantumKVCache.js'; +import type { RedisKVCacheOpts, RedisSingleCacheOpts, MemoryCacheOpts } from '@/misc/cache.js'; +import type { TimeService } from '@/global/TimeService.js'; +import type { InternalEventService } from '@/global/InternalEventService.js'; +import { + CacheManagementService, + type ManagedMemoryKVCache, + type ManagedMemorySingleCache, + type ManagedRedisKVCache, + type ManagedRedisSingleCache, + type ManagedQuantumKVCache, +} from '@/global/CacheManagementService.js'; + +/** + * Fake implementation of cache management that suppresses all caching behavior. + * The returned cache instances are real and fully functional, but expiration is negative to ensure that data is immediately discarded and nothing is cached. + * Essentially, it strips out the caching behavior and converts caches into pure data accessors. + */ +@Injectable() +export class FakeCacheManagementService extends CacheManagementService { + constructor(opts?: { + redisClient?: Redis.Redis; + timeService?: TimeService; + internalEventService?: InternalEventService; + }) { + const timeService = opts?.timeService ?? new GodOfTimeService(); + const redisClient = opts?.redisClient ?? new MockRedis(timeService); + const internalEventService = opts?.internalEventService ?? new MockInternalEventService(); + + super(redisClient, timeService, internalEventService); + } + + createMemoryKVCache(name: string, optsOrLifetime: number | MemoryCacheOpts): ManagedMemoryKVCache { + const opts = typeof(optsOrLifetime) === 'number' ? { lifetime: -1 } : { ...optsOrLifetime, lifetime: -1 }; + return super.createMemoryKVCache(name, opts); + } + + createMemorySingleCache(name: string, optsOrLifetime: number | MemoryCacheOpts): ManagedMemorySingleCache { + const opts = typeof(optsOrLifetime) === 'number' ? { lifetime: -1 } : { ...optsOrLifetime, lifetime: -1 }; + return super.createMemorySingleCache(name, opts); + } + + createRedisKVCache(name: string, opts: RedisKVCacheOpts): ManagedRedisKVCache { + return super.createRedisKVCache(name, { + ...opts, + lifetime: -1, + memoryCacheLifetime: -1, + }); + } + + createRedisSingleCache(name: string, opts: RedisSingleCacheOpts): ManagedRedisSingleCache { + return super.createRedisSingleCache(name, { + ...opts, + lifetime: -1, + memoryCacheLifetime: -1, + }); + } + + createQuantumKVCache(name: string, opts: QuantumKVOpts): ManagedQuantumKVCache { + return super.createQuantumKVCache(name, { + ...opts, + lifetime: -1, + }); + } +} diff --git a/packages/backend/test/misc/FakeRedis.ts b/packages/backend/test/misc/FakeRedis.ts new file mode 100644 index 0000000000..e26962760f --- /dev/null +++ b/packages/backend/test/misc/FakeRedis.ts @@ -0,0 +1,146 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Redis as RedisConstructor } from 'ioredis'; +import type * as Redis from 'ioredis'; + +export type RedisKey = Redis.RedisKey; +export type RedisString = Buffer | string; +export type RedisNumber = string | number; +export type RedisValue = RedisKey | RedisString | RedisNumber; +export type RedisCallback = Redis.Callback; + +export type Ok = 'OK'; +export const ok = 'OK' as const; + +export type FakeRedis = RedisConstructor; +export interface FakeRedisConstructor { + new (): FakeRedis; +} + +/** + * Fake implementation of Redis that pretends to connect but throws on any operation. + */ +export const FakeRedis: FakeRedisConstructor = createFakeRedis(); + +function createFakeRedis(): FakeRedisConstructor { + class FakeRedis implements Partial { + async connect(callback?: RedisCallback): Promise { + // no-op + callback?.(null); + } + + async hello(...callbacks: (undefined | string | number | Buffer | RedisCallback)[]): Promise { + // no-op + const callback = callbacks.find(c => typeof(c) === 'function'); + callback?.(null, []); + return []; + } + + async auth(...callbacks: (undefined | string | Buffer | RedisCallback)[]): Promise { + const callback = callbacks.find(c => typeof(c) === 'function'); + callback?.(null, ok); + return ok; + } + + async quit(callback?: RedisCallback) { + // no-op + callback?.(null, ok); + return ok; + } + + async save(callback?: RedisCallback) { + // no-op + callback?.(null, ok); + return ok; + } + + async sync(callback?: RedisCallback) { + // no-op + callback?.(null, ok); + return ok; + } + + disconnect(): void { + // no-op + } + + end(): void { + // no-op + } + } + + const fakeProto = FakeRedis.prototype as Partial; + const redisProto = RedisConstructor.prototype as Partial; + + // Override all methods and accessors from Redis + for (const [key, property] of allProps(redisProto)) { + // Skip anything already defined + if (Reflect.has(fakeProto, key)) { + continue; + } + + if (property.get || property.set) { + // Stub accessors + Reflect.defineProperty(fakeProto, key, { + ...property, + get: property.get ? stub(property.get.name || key) : undefined, + set: property.set ? stub(property.set.name || key) : undefined, + }); + } else if (property.value && typeof(property.value) === 'function') { + // Stub methods + Reflect.defineProperty(fakeProto, key, { + ...property, + value: stub(property.value.name || key), + }); + } + } + + // Fixup protoype + Reflect.setPrototypeOf(fakeProto, redisProto); + + // test + const test = new FakeRedis(); + if (!(test instanceof RedisConstructor)) { + throw new Error('failed to extend'); + } + + return FakeRedis as FakeRedisConstructor; +} + +function *allProps(obj: object | null): Generator<[PropertyKey, PropertyDescriptor]> { + while (obj != null) { + for (const key of Reflect.ownKeys(obj)) { + const prop = Reflect.getOwnPropertyDescriptor(obj, key); + if (prop) { + yield [key, prop]; + } + } + + obj = Object.getPrototypeOf(obj); + } +} + +function stub(name: PropertyKey) { + if (typeof(name) === 'symbol') { + name = `[symbol.${name.description || ''}]`; + } else if (typeof(name) === 'number') { + name = String(name); + } + + const stub = () => { + throw new Error(`Not Implemented: MockRedis does not support ${name}`); + }; + + // Make the stub match the original name + Object.defineProperty(stub, 'name', { + writable: false, + enumerable: false, + configurable: true, + value: name, + }); + + return stub; +} diff --git a/packages/backend/test/misc/FakeSkRateLimiterService.ts b/packages/backend/test/misc/FakeSkRateLimiterService.ts new file mode 100644 index 0000000000..7cd730a1e3 --- /dev/null +++ b/packages/backend/test/misc/FakeSkRateLimiterService.ts @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import type { LimitInfo } from '@/misc/rate-limit-utils.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; + +/** + * Fake implementation of SkRateLimiterService that does not enforce any limits. + */ +@Injectable() +export class FakeSkRateLimiterService extends SkRateLimiterService { + async limit(): Promise { + return { + blocked: false, + remaining: Number.MAX_SAFE_INTEGER, + resetMs: 0, + resetSec: 0, + fullResetMs: 0, + fullResetSec: 0, + }; + } +} diff --git a/packages/backend/test/misc/GodOfTimeService.ts b/packages/backend/test/misc/GodOfTimeService.ts new file mode 100644 index 0000000000..61e267a52d --- /dev/null +++ b/packages/backend/test/misc/GodOfTimeService.ts @@ -0,0 +1,131 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import { TimeService, type Timer } from '@/global/TimeService.js'; +import { addPatch, type DatePatch } from '@/misc/patch-date.js'; + +/** + * Fake implementation of TimeService that allows manual control of time. + * When this service is used, the flow of time is fully stopped. + * + * Test cases can manually adjust the "now" parameter to move time forwards and backwards. + * When moving forward, timers (interval and timeout) will automatically fire as appropriate. + */ +@Injectable() +export class GodOfTimeService extends TimeService { + private _now = 0; + + constructor() { + super(); + } + + /** + * Get or set the current time, in milliseconds since the unix epoch. + */ + public get now() { + return this._now; + } + public set now(value: number) { + // Moving backwards is allowed, for now. + if (value > this._now) { + // Since timers may repeat, we need to loop this. + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + while (true) { + // Fire all expiring timers in chronological order. + const expiringTimers = this.timers + .values() + .filter(t => t.expiresAt <= value) + .toArray() + .sort((a, b) => a.expiresAt - b.expiresAt); + + // Stop when everything is caught up + if (expiringTimers.length === 0) { + break; + } + + // Since we sorted the list, this will progressively increase "now" as we handle later and later events. + for (const timer of expiringTimers) { + // When the timer fires, "now" should equal the time that was originally waited for. + this._now = timer.expiresAt; + this.runTimer(timer); + } + } + } + + // Bump up to the final target value + this._now = value; + } + + private runTimer(timer: GodsOwnTimer): void { + // Cleanup first in case timer throws an exception. + if (timer.repeating) { + timer.expiresAt = this._now + timer.delay; + } else { + this.timers.delete(timer.timerId); + } + + // Fire the actual callback. + // If it throws an error, then processing will stop halfway. + // This is good, since it means the adjustment can be retried safely. + timer.callback(); + } + + /** + * Get or set the current time, as a JavaScript Date object. + */ + get date(): Date { + return super.date; + } + set date(value: Date) { + this.now = value.getTime(); + } + + /** + * Moves time by a relative "tick" amount. + * Ticks can be a raw number of milliseconds, or an inline object containing time and/or date increments. + */ + public tick(tick: number | DatePatch) { + if (typeof(tick) === 'number') { + this.now += tick; + } else { + this.date = addPatch(this.date, tick); + } + } + + /** + * Clears all timers and resets to time=0. + */ + public reset() { + this.resetTo(0); + } + /** + * Clears all timers and resets to the real-world time. + */ + public resetToNow() { + this.resetTo(Date.now()); + } + + /** + * Clears all timers and resets to a given time. + */ + public resetTo(to: number) { + this.timers.clear(); + this.now = to; + } + + protected startNativeTimer(timerId: symbol, repeating: boolean, callback: () => void, delay: number): GodsOwnTimer { + const expiresAt = this.now + delay; + return { timerId, repeating, delay, expiresAt, callback }; + } + + protected stopNativeTimer(): void { + // no-op - fake timers have no side effects to clean up + } +} + +export interface GodsOwnTimer extends Timer { + expiresAt: number; +} diff --git a/packages/backend/test/misc/MockApResolverService.ts b/packages/backend/test/misc/MockApResolverService.ts new file mode 100644 index 0000000000..c066c3c004 --- /dev/null +++ b/packages/backend/test/misc/MockApResolverService.ts @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import { MockResolver } from './mock-resolver.js'; +import type { Config } from '@/config.js'; +import type { MiMeta } from '@/models/Meta.js'; +import type { + UsersRepository, + NotesRepository, + PollsRepository, + NoteReactionsRepository, + FollowRequestsRepository, +} from '@/models/_.js'; +import { ApResolverService } from '@/core/activitypub/ApResolverService.js'; +import { DI } from '@/di-symbols.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { SystemAccountService } from '@/core/SystemAccountService.js'; +import { ApRequestService } from '@/core/activitypub/ApRequestService.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; +import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import { ApLogService } from '@/core/ApLogService.js'; +import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; +import { CacheService } from '@/core/CacheService.js'; +import { bindThis } from '@/decorators.js'; + +/** + * Mock implementation of ApResolverService to automatically provide a MockResolver to the entire test environment. + */ +@Injectable() +export class MockApResolverService extends ApResolverService { + /** + * Resolver that will be provided. + */ + public readonly resolver: MockResolver; + + constructor( + @Inject(DI.config) + config: Config, + + @Inject(DI.meta) + meta: MiMeta, + + @Inject(DI.usersRepository) + usersRepository: UsersRepository, + + @Inject(DI.notesRepository) + notesRepository: NotesRepository, + + @Inject(DI.pollsRepository) + pollsRepository: PollsRepository, + + @Inject(DI.noteReactionsRepository) + noteReactionsRepository: NoteReactionsRepository, + + @Inject(DI.followRequestsRepository) + followRequestsRepository: FollowRequestsRepository, + + utilityService: UtilityService, + systemAccountService: SystemAccountService, + apRequestService: ApRequestService, + httpRequestService: HttpRequestService, + apRendererService: ApRendererService, + apDbResolverService: ApDbResolverService, + loggerService: LoggerService, + apLogService: ApLogService, + apUtilityService: ApUtilityService, + cacheService: CacheService, + ) { + super( + config, + meta, + usersRepository, + notesRepository, + pollsRepository, + noteReactionsRepository, + followRequestsRepository, + utilityService, + systemAccountService, + apRequestService, + httpRequestService, + apRendererService, + apDbResolverService, + loggerService, + apLogService, + apUtilityService, + cacheService, + ); + + this.resolver = new MockResolver( + this.config, + this.meta, + this.usersRepository, + this.notesRepository, + this.pollsRepository, + this.noteReactionsRepository, + this.followRequestsRepository, + this.utilityService, + this.systemAccountService, + this.apRequestService, + this.httpRequestService, + this.apRendererService, + this.apDbResolverService, + this.loggerService, + this.apLogService, + this.apUtilityService, + this.cacheService, + ); + } + + /** + * Resets the mock to initial state. + */ + @bindThis + reset() { + this.resolver.clear(); + } + + @bindThis + createResolver(): MockResolver { + return this.resolver; + } +} diff --git a/packages/backend/test/misc/MockConsole.ts b/packages/backend/test/misc/MockConsole.ts new file mode 100644 index 0000000000..f307ee2fc9 --- /dev/null +++ b/packages/backend/test/misc/MockConsole.ts @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { jest } from '@jest/globals'; +import { Injectable } from '@nestjs/common'; +import { bindThis } from '@/decorators.js'; + +/** + * Console implementation where all members are jest mocks. + */ +@Injectable() +export class MockConsole implements Console { + public readonly Console = MockConsole; + + /** + * Resets all mocks in the console. + */ + @bindThis + public mockReset(): void { + for (const func of Object.values(this)) { + if (typeof(func) === 'function' && 'mockReset' in func) { + func.mockReset(); + } + } + } + + /** + * Asserts that no errors and/or warnings have been logged. + */ + @bindThis + public assertNoErrors(opts?: { orWarnings?: boolean }): void { + expect(this.error).not.toHaveBeenCalled(); + + if (opts?.orWarnings) { + expect(this.warn).not.toHaveBeenCalled(); + } + } + + public readonly error = jest.fn(); + public readonly warn = jest.fn(); + public readonly info = jest.fn(); + public readonly log = jest.fn(); + public readonly debug = jest.fn(); + public readonly trace = jest.fn(); + public readonly assert = jest.fn(); + public readonly clear = jest.fn(); + public readonly count = jest.fn(); + public readonly countReset = jest.fn(); + public readonly dir = jest.fn(); + public readonly dirxml = jest.fn(); + public readonly group = jest.fn(); + public readonly groupCollapsed = jest.fn(); + public readonly groupEnd = jest.fn(); + public readonly table = jest.fn(); + public readonly time = jest.fn(); + public readonly timeEnd = jest.fn(); + public readonly timeLog = jest.fn(); + public readonly profile = jest.fn(); + public readonly profileEnd = jest.fn(); + public readonly timeStamp = jest.fn(); +} diff --git a/packages/backend/test/misc/MockDependencyService.ts b/packages/backend/test/misc/MockDependencyService.ts new file mode 100644 index 0000000000..a8a7f2637e --- /dev/null +++ b/packages/backend/test/misc/MockDependencyService.ts @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import { DependencyService } from '@/global/DependencyService.js'; +import { bindThis } from '@/decorators.js'; + +/** + * Extension of DependencyService that allows version information to be mocked. + */ +@Injectable() +export class MockDependencyService extends DependencyService { + /** + * Overrides the version for a dependency. + * Pass a string or null to override the version, or pass undefined to clear the override and restore the original value. + */ + @bindThis + public setDependencyVersion(dependency: string, version: string | null | undefined) { + if (version !== undefined) { + this.dependencyVersionCache.set(dependency, version); + } else { + this.dependencyVersionCache.delete(dependency); + } + } + + /** + * Resets the mock to initial values. + */ + @bindThis + public mockReset(): void { + this.dependencyVersionCache.clear(); + } +} diff --git a/packages/backend/test/misc/MockEnvService.ts b/packages/backend/test/misc/MockEnvService.ts new file mode 100644 index 0000000000..d3f3d5b290 --- /dev/null +++ b/packages/backend/test/misc/MockEnvService.ts @@ -0,0 +1,102 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import process from 'node:process'; +import { Injectable } from '@nestjs/common'; +import { EnvService } from '@/global/EnvService.js'; +import { bindThis } from '@/decorators.js'; + +/** + * Implementation of EnvService with support for mocking values. + * Environment and package versions are loaded from their original sources, but can be overridden as-needed. + */ +@Injectable() +export class MockEnvService extends EnvService { + public readonly _env: typeof process['env']; + + private overrides: Partial> = {}; + + constructor() { + super(); + this._env = new Proxy(process.env, { + get: (env, key) => { + if (key in this.overrides && this.overrides[key as string] !== undefined) { + return this.overrides[key as string] ?? undefined; + } else { + return env[key as string]; + } + }, + has: (env, key) => { + if (key in this.overrides && this.overrides[key as string] !== undefined) { + return this.overrides[key as string] != null; + } else { + return key in env; + } + }, + set: (_, key, value) => { + this.overrides[key as string] = value; + return true; + }, + deleteProperty: (_, key) => { + this.overrides[key as string] = null; + return true; + }, + ownKeys: (env) => { + const envKeys = Reflect.ownKeys(env); + const allKeys = new Set(envKeys); + + const overrides = Object.entries(this.overrides); + for (const [key, value] of overrides) { + if (value !== undefined) { + if (value === null) { + allKeys.delete(key); + } else { + allKeys.add(key); + } + } + } + + return Array.from(allKeys); + }, + }); + } + + /** + * Gets the mocked environment. + * The returned object is "live" and can be modified without polluting the actual application environment. + */ + get env(): Partial> { + return this._env; + } + + /** + * Returns a variable from the mocked environment. + */ + public get(key: string): string | undefined { + return this.env[key]; + } + + /** + * Sets a variable in the mocked environment. + */ + public set(key: string, value: string): void { + this.overrides[key] = value; + } + + /** + * Removes a variable from the mocked environment. + */ + public delete(key: string): void { + this.overrides[key] = null; + } + + /** + * Resets the mock to initial values. + */ + @bindThis + public mockReset(): void { + this.overrides = {}; + } +} diff --git a/packages/backend/test/misc/FakeInternalEventService.ts b/packages/backend/test/misc/MockInternalEventService.ts similarity index 77% rename from packages/backend/test/misc/FakeInternalEventService.ts rename to packages/backend/test/misc/MockInternalEventService.ts index d18a080eaf..79b455cf11 100644 --- a/packages/backend/test/misc/FakeInternalEventService.ts +++ b/packages/backend/test/misc/MockInternalEventService.ts @@ -3,10 +3,12 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import type { Listener, ListenerProps } from '@/core/InternalEventService.js'; -import type Redis from 'ioredis'; -import type { GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; +import { Injectable } from '@nestjs/common'; +import { MockRedis } from './MockRedis.js'; +import type { Listener, ListenerProps } from '@/global/InternalEventService.js'; +import type { InternalEventTypes } from '@/core/GlobalEventService.js'; +import type { Config } from '@/config.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; import { bindThis } from '@/decorators.js'; type FakeCall = [K, Parameters]; @@ -17,7 +19,8 @@ type FakeListener = [K, Listener, Listene * There is no redis connection, and metadata is tracked in the public _calls and _listeners arrays. * The on/off/emit methods are fully functional and can be called in tests to invoke any registered listeners. */ -export class FakeInternalEventService extends InternalEventService { +@Injectable() +export class MockInternalEventService extends InternalEventService { /** * List of calls to public methods, in chronological order. */ @@ -32,7 +35,7 @@ export class FakeInternalEventService extends InternalEventService { * Resets the mock. * Clears all listeners and tracked calls. */ - public _reset() { + public mockReset() { this._calls = []; this._listeners = []; } @@ -41,15 +44,15 @@ export class FakeInternalEventService extends InternalEventService { * Simulates a remote event sent from another process in the cluster via redis. */ @bindThis - public async _emitRedis(type: K, value: InternalEventTypes[K]): Promise { + public async mockEmit(type: K, value: InternalEventTypes[K]): Promise { await this.emit(type, value, false); } - constructor() { - super( - { on: () => {} } as unknown as Redis.Redis, - {} as unknown as GlobalEventService, - ); + constructor( + config?: Pick, + ) { + const redis = new MockRedis(); + super(redis, redis, config ?? { host: 'example.com' }); } @bindThis diff --git a/packages/backend/test/misc/MockRedis.ts b/packages/backend/test/misc/MockRedis.ts new file mode 100644 index 0000000000..6e0796da3c --- /dev/null +++ b/packages/backend/test/misc/MockRedis.ts @@ -0,0 +1,790 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import { FakeRedis, ok, type RedisString } from './FakeRedis.js'; +import type { RedisKey, RedisNumber, RedisValue, RedisCallback, Ok } from './FakeRedis.js'; +import type { ChainableCommander } from 'ioredis'; +import { TimeService, NativeTimeService } from '@/global/TimeService.js'; +import { bindThis } from '@/decorators.js'; + +export interface MockRedisConstructor { + new (timeService?: TimeService): MockRedis; +} + +export interface MockRedis extends FakeRedis { + /** + * Gets a key/value entry from the mock, with metadata. + * Returns undefined if no value is stored. + */ + mockGetEntry(key: RedisKey): MockEntry | undefined; + + /** + * Gets a value from the mock, or undefined if no value is stored. + */ + mockGet(key: RedisKey): RedisValue | undefined; + + /** + * Deletes a value from the mock. + * Does nothing if the value doesn't exist. + */ + mockDel(key: RedisKey): void; + + /** + * Sets a value in the mock, replacing any prior value. + * Expiration, if provided, should be in milliseconds since the Unix epoch. + */ + mockSet(key: RedisKey, value: RedisValue, expiration?: number | null): void; + + /** + * Resets the mock to initial state. + */ + mockReset(): void; +} + +export interface MockEntry { + key: RedisKey; + value: RedisValue; + expiration: number | null; +} + +/** TODO implement the other commands */ +class MockTransactionImpl extends FakeRedis { + private readonly commands: [keyof MockRedis, ...unknown[]][]; + + get length() { + return this.commands.length; + } + + constructor( + private readonly mockRedis: MockRedis, + commands: unknown[][] = [], + ) { + super(); + this.commands = commands.map(([command, ...args]) => ([ + String(command).toLowerCase() as keyof MockRedis, + ...args, + ])); + } + + @bindThis + public async exec(callback?: RedisCallback<[error: Error | null, result: unknown][] | null>): Promise<[error: Error | null, result: unknown][] | null> { + const results: [error: Error | null, result: unknown][] = []; + + for (const [command, ...args] of this.commands) { + try { + const res = await (this.mockRedis[command] as (...args: unknown[]) => Promise)(...args); + results.push([null, res]); + } catch (err) { + const error = err instanceof Error ? err : new Error('Unknown error', { cause: err }); + results.push([error, undefined]); + } + } + + callback?.(null, results); + return results; + } +} + +const MockTransaction = MockTransactionImpl as unknown as { + new (mockRedis: unknown, commands?: unknown[][]): ChainableCommander; +}; + +/** + * Mock implementation of Redis that works in-memory and exposes functions to manipulate the values. + * Throws on any unsupported operation, and never actually connects. + */ +export const MockRedis: MockRedisConstructor = createMockRedis(); + +function createMockRedis(): MockRedisConstructor { + @Injectable() + class MockRedis extends FakeRedis implements MockRedis { + private readonly timeService: TimeService; + private readonly mockData = new Map(); // redis data + private readonly mockEvents = new EventManager(); // on/of/once listeners + private readonly mockChannels = new Set(); // subscribed pub/sub channels + + constructor(timeService?: TimeService) { + super(); + this.timeService = timeService ?? new NativeTimeService(); + } + + @bindThis + mockGetEntry(key: RedisKey): MockEntry | undefined { + const mapped = mapKey(key); + + let entry = this.mockData.get(mapped); + if (entry?.expiration && entry.expiration <= this.timeService.now) { + this.mockDel(key); + entry = undefined; + } + + return entry; + } + + @bindThis + mockGet(key: RedisKey): RedisValue | undefined { + const entry = this.mockGetEntry(key); + return entry?.value; + } + + @bindThis + mockDel(key: RedisKey): void { + const mapped = mapKey(key); + this.mockData.delete(mapped); + } + + @bindThis + mockSet(key: RedisKey, value: RedisValue, expiration: number | null = null): void { + const mapped = mapKey(key); + this.mockData.set(mapped, { + expiration, + value, + key, + }); + } + + @bindThis + mockReset(): void { + this.mockChannels.clear(); + this.mockEvents.clear(); + this.mockData.clear(); + } + + @bindThis + dispose(): void { + this.mockEvents.dispose(); + this.mockReset(); + } + + @bindThis + public once(ev: string, callback: AnyCallback): this { + this.mockEvents.once(ev, callback); + return this; + } + + @bindThis + public on(ev: string, callback: AnyCallback): this { + this.mockEvents.on(ev, callback); + return this; + } + + @bindThis + public off(ev: string, callback: AnyCallback): this { + this.mockEvents.off(ev, callback); + return this; + } + + @bindThis + public async subscribe(...args: (RedisString | RedisCallback)[]): Promise { + const callback = args + .find(a => typeof(a) === 'function'); + const channels = args + .filter(a => typeof(a) !== 'function') + .flat() + .map(s => parseString(s)); + + for (const channel of channels) { + this.mockChannels.add(channel); + } + + callback?.(null, ok); + return ok; + } + + @bindThis + public async unsubscribe(...args: (RedisString | RedisCallback | undefined)[]): Promise { + const callback = args + .find(a => typeof(a) === 'function'); + const channels = args + .filter(a => typeof(a) !== 'function') + .flat() + .filter(s => s != null) + .map(s => parseString(s)); + + for (const channel of channels) { + this.mockChannels.delete(channel); + } + + callback?.(null, ok); + return ok; + } + + @bindThis + public async publish(channel: RedisString, message: RedisString, callback?: RedisCallback): Promise { + const channelString = parseString(channel); + if (!this.mockChannels.has(channelString)) { + callback?.(null, 0); + return 0; + } + + if (Buffer.isBuffer(message)) { + channel = Buffer.from(channel); + this.mockEvents.emit('messageBuffer', channel, message); + } else { + message = parseString(message); + this.mockEvents.emit('message', channelString, message); + } + + callback?.(null, 1); + return 1; + } + + @bindThis + public pipeline(commands?: unknown[][]): ChainableCommander { + return this.multi(commands); + } + + public multi(options: { pipeline: false }): Promise; + public multi(options: { pipeline: true }): ChainableCommander; + public multi(commands?: unknown[][]): ChainableCommander; + @bindThis + public multi(commandsOrOptions?: unknown[][] | { pipeline: boolean }): Promise | ChainableCommander { + if (Array.isArray(commandsOrOptions)) { + return new MockTransaction(this, commandsOrOptions); + } else if (commandsOrOptions == null || commandsOrOptions.pipeline) { + return new MockTransaction(this); + } else { + return Promise.resolve(ok); + } + } + + @bindThis + public async get(key: RedisKey, callback?: RedisCallback): Promise { + let value = this.mockGet(key); + + // Emulate implicit casts + if (typeof(value) === 'number') { + value = String(value); + } + + if (value != null && typeof(value) !== 'string') { + const err = new Error('get failed: cannot GET a non-string value'); + callback?.(err); + throw err; + } + + callback?.(null, value ?? null); + return value ?? null; + } + + @bindThis + public async getBuffer(key: RedisKey, callback?: RedisCallback): Promise { + let value = this.mockGet(key); + + // Emulate implicit casts + if (typeof(value) === 'number') { + value = String(value); + } + + if (value != null && !Buffer.isBuffer(value)) { + const err = new Error('getBuffer failed: cannot GET a non-buffer value'); + callback?.(err); + throw err; + } + + callback?.(null, value ?? null); + return value ?? null; + } + + @bindThis + public async getDel(key: RedisKey, callback?: RedisCallback): Promise { + let value = this.mockGet(key); + + // Emulate implicit casts + if (typeof(value) === 'number') { + value = String(value); + } + + if (value != null && typeof(value) !== 'string') { + const err = new Error('getDel failed: cannot GETDEL a non-string value'); + callback?.(err); + throw err; + } + + this.mockDel(key); + + callback?.(null, value ?? null); + return value ?? null; + } + + @bindThis + public async getDelBuffer(key: RedisKey, callback?: RedisCallback): Promise { + const value = this.mockGet(key); + + if (value != null && !Buffer.isBuffer(value)) { + const err = new Error('getDelBuffer failed: cannot GETDEL a non-string value'); + callback?.(err); + throw err; + } + + this.mockDel(key); + + callback?.(null, value ?? null); + return value ?? null; + } + + @bindThis + public async getSet(key: RedisKey, newValue: RedisValue, callback?: RedisCallback): Promise { + const oldValue = this.mockGet(key); + + if (oldValue != null && typeof(oldValue) !== 'string') { + const err = new Error('getSet failed: cannot GETSET a non-string value'); + callback?.(err); + throw err; + } + + this.mockSet(key, newValue); + + callback?.(null, oldValue ?? null); + return oldValue ?? null; + } + + @bindThis + public async getSetBuffer(key: RedisKey, newValue: RedisValue, callback?: RedisCallback): Promise { + const oldValue = this.mockGet(key); + + if (oldValue != null && !Buffer.isBuffer(oldValue)) { + const err = new Error('getSetBuffer failed: cannot GETSET a non-string value'); + callback?.(err); + throw err; + } + + this.mockSet(key, newValue); + + callback?.(null, oldValue ?? null); + return oldValue ?? null; + } + + @bindThis + public async del(...args: (RedisKey | RedisKey[] | RedisCallback | undefined)[]): Promise { + const callback = args.find(a => typeof(a) === 'function'); + const keys = args.filter(a => typeof(a) !== 'function').flat(); + + let total = 0; + for (const key of keys) { + if (key == null) { + continue; + } + + const entry = this.mockGet(key); + if (entry) { + total++; + this.mockDel(key); + } + } + + callback?.(null, total); + return total; + } + + @bindThis + public async incr(key: RedisKey, callback?: RedisCallback): Promise { + return await this.incrCommon(key, 1, true, 'incr', callback); + } + + @bindThis + public async incrby(key: RedisKey, increment: RedisNumber, callback?: RedisCallback): Promise { + return await this.incrCommon(key, increment, true, 'incrby', callback); + } + + @bindThis + public async decr(key: RedisKey, callback?: RedisCallback): Promise { + return await this.incrCommon(key, 1, false, 'decr', callback); + } + + @bindThis + public async decrby(key: RedisKey, increment: RedisNumber, callback?: RedisCallback): Promise { + return await this.incrCommon(key, increment, false, 'decrby', callback); + } + + @bindThis + private async incrCommon(key: RedisKey, increment: RedisNumber, add: boolean, func: string, callback?: RedisCallback): Promise { + // Parse the increment + const inc = parseNumber(increment); + if (inc == null) { + const err = new Error(`${func} failed: cannot parse increment as integer`); + callback?.(err); + throw err; + } + + // Extract and verify the value + const entry = this.mockGetEntry(key); + let value = entry != null ? parseNumber(entry.value) : 0; + if (value == null) { + const err = new Error(`${func} failed: cannot ${func.toUpperCase()} a non-number value`); + callback?.(err); + throw err; + } + + // Apply the increment + if (add) { + value += inc; + } else { + value -= inc; + } + + // Update, but preserve expiration + this.mockSet(key, value, entry?.expiration); + + callback?.(null, value); + return value; + } + + expire(key: RedisKey, seconds: RedisNumber, callback?: RedisCallback): Promise; + expire(key: RedisKey, seconds: RedisNumber, flag: 'NX', callback?: RedisCallback): Promise; + expire(key: RedisKey, seconds: RedisNumber, flag: 'XX', callback?: RedisCallback): Promise; + expire(key: RedisKey, seconds: RedisNumber, flag: 'GT', callback?: RedisCallback): Promise; + expire(key: RedisKey, seconds: RedisNumber, flag: 'LT', callback?: RedisCallback): Promise; + @bindThis + public async expire(key: RedisKey, seconds: RedisNumber, callbackOrFlag?: RedisCallback | 'NX' | 'XX' | 'GT' | 'LT', orCallback?: RedisCallback): Promise { + const flag = typeof(callbackOrFlag) === 'string' ? callbackOrFlag : null; + const callback = typeof(callbackOrFlag) === 'function' ? callbackOrFlag : orCallback; + + const expiresSec = parseNumber(seconds); + if (expiresSec == null) { + const err = new Error('expire failed: cannot parse seconds as integer'); + callback?.(err); + throw err; + } + + // Non-positive expires should execute DEL instead. + // https://redis.io/docs/latest/commands/expire + if (expiresSec < 1) { + return await this.del(key, callback); + } + + const entry = this.mockGetEntry(key); + if (!entry) { + callback?.(null, 0); + return 0; + } + + if (flag === 'NX' && entry.expiration != null) { + callback?.(null, 0); + return 0; + } + + if (flag === 'XX' && entry.expiration == null) { + callback?.(null, 0); + return 0; + } + + const expiresAt = this.timeService.now + (expiresSec * 1000); + if (entry.expiration != null) { + if (flag === 'GT' && expiresAt <= entry.expiration) { + callback?.(null, 0); + return 0; + } + + if (flag === 'LT' && expiresAt >= entry.expiration) { + callback?.(null, 0); + return 0; + } + } + + // Success! update it + entry.expiration = expiresAt; + callback?.(null, 1); + return 1; + } + + @bindThis + public async setex(key: RedisKey, seconds: RedisNumber, value: RedisValue, callback?: RedisCallback): Promise { + await this.set(key, value, 'EX', seconds); + callback?.(null, ok); + return ok; + } + + @bindThis + public async setnx(key: RedisKey, value: RedisValue, callback?: RedisCallback): Promise { + const ok = await this.set(key, value, 'NX'); + callback?.(null, ok ? 1 : 0); + return ok ? 1 : 0; + } + + @bindThis + // @ts-expect-error This comes from collapsing all the overload signatures, but it's fine. + public async set( + key: RedisKey, + value: RedisValue, + op1?: SetOp1 | RedisCallback, + op2?: SetOp2 | SetArg | RedisCallback, + op3?: SetOp3 | SetArg | RedisCallback, + op4?: SetArg | RedisCallback, + op5?: RedisCallback, + ): Promise { + const entry = this.mockGetEntry(key); + + // Parse ops + const { nx, ex, get, cb, err } = this._parseSetOps(entry ?? null, [op1, op2, op3, op4, op5]); + + // Additional error from the "GET" flag + if (get && entry != null && typeof(entry.value) !== 'string') { + err.push(new Error('set failed: cannot GET a non-string value.')); + } + + // Abort on errors + if (err.length > 1) { + const agg = new AggregateError(err, 'set failed: see "errors" property for details.'); + if (cb) cb(agg); + throw agg; + } else if (err.length > 0) { + if (cb) cb(err[0]); + throw err[0]; + } + + // Emulate the "NX" and "XX" flags + const nxLock = + (nx === true && entry != null) || // NX - skip if the key already exists (Never eXchange) + (nx === false && entry == null); // XX - skip the key *doesn't* exist (eXclusively eXchange) + + // Compute return value for the operation + const ret = get + ? entry // Return the previous value or null for GET + ? parseString(entry.value) + : null + : nxLock + ? null // Return null if locked out by NX or XX + : ok; // Otherwise return ok, even if we don't set it. + + // Write *after* we compute the return value! + const doWrite = !nxLock; + if (doWrite) { + this.mockSet(key, value, ex); + } + + // Return the results + if (cb) cb(null, ret); + return ret; + } + + @bindThis + private _parseSetOps(entry: MockEntry | null, ops: SetOp[]) { + const err: Error[] = []; + let ex: number | null | undefined; + let nx: boolean | null = null; + let get = false; + let cb: RedisCallback | null = null; + + // Slide through it til we reach the end + let nextIsParam = false; + for (let i = 0; i < ops.length && ops[i] != null; i++) { + // This is set when one of the ops consumed the next token as an argument. + if (nextIsParam) { + nextIsParam = false; + continue; + } + + const opRaw = ops[i]; + const op = typeof(opRaw) === 'function' ? opRaw : parseString(opRaw); + + const argRaw = ops[i + 1]; + const arg = typeof(argRaw) === 'function' ? argRaw : parseNumber(argRaw); + + if (typeof(op) === 'function') { + cb = op as RedisCallback; + } else if (op === 'KEEPTTL') { + ex = entry?.expiration; + } else if (op === 'GET') { + get = true; + } else if (op === 'NX') { + nx = true; + } else if (op === 'XX') { + nx = false; + } else if (op === 'EX') { + nextIsParam = true; + if (arg == null) { + err.push(new Error('Missing required argument for set "EX" parameter')); + } else if (typeof(arg) !== 'number') { + err.push(new Error('Invalid argument for set "EX" parameter: must be a number')); + } else if (arg < 0) { + err.push(new Error('Invalid argument for set "EX" parameter: must be a positive integer')); + } else { + ex = this.timeService.now + (arg * 1000); + } + } else if (op === 'PX') { + nextIsParam = true; + if (arg == null) { + err.push(new Error('Missing required argument for set "EX" parameter')); + } else if (typeof(arg) !== 'number') { + err.push(new Error('Invalid argument for set "EX" parameter: must be a number')); + } else if (arg < 0) { + err.push(new Error('Invalid argument for set "EX" parameter: must be a positive integer')); + } else { + ex = this.timeService.now + arg; + } + } else if (op === 'EXAT') { + nextIsParam = true; + if (arg == null) { + err.push(new Error('Missing required argument for set "EX" parameter')); + } else if (typeof(arg) !== 'number') { + err.push(new Error('Invalid argument for set "EX" parameter: must be a number')); + } else if (arg < 0) { + err.push(new Error('Invalid argument for set "EX" parameter: must be a positive integer')); + } else { + ex = arg * 1000; + } + } else if (op === 'PXAT') { + nextIsParam = true; + if (arg == null) { + err.push(new Error('Missing required argument for set "EX" parameter')); + } else if (typeof(arg) !== 'number') { + err.push(new Error('Invalid argument for set "EX" parameter: must be a number')); + } else if (arg < 0) { + err.push(new Error('Invalid argument for set "EX" parameter: must be a positive integer')); + } else { + ex = arg; + } + } else { + err.push(new Error(`Unknown parameter for set: "${op}"`)); + } + } + + return { ex, nx, get, cb, err }; + } + } + + return MockRedis as unknown as MockRedisConstructor; +} + +function mapKey(key: RedisKey): string { + const prefix = Buffer.isBuffer(key) ? 'b' : 's'; + const mapped = parseString(key); + return `${prefix}:${mapped}`; +} + +function parseNumber(value: RedisValue | undefined): number | undefined; +function parseNumber(value: RedisValue | null | undefined): number | null | undefined; + +function parseNumber(value: RedisValue | null | undefined): number | null | undefined { + if (value == null) { + return value; + } + + if (typeof(value) !== 'number') { + value = parseString(value); + value = parseInt(value); + } + + if (!Number.isSafeInteger(value)) { + return undefined; + } + + if (Number.isNaN(value)) { + return undefined; + } + + return value; +} + +function parseString(value: RedisValue): string; +function parseString(value: RedisValue | null): string | null; +function parseString(value: RedisValue | undefined): string | undefined; +function parseString(value: RedisValue | null | undefined): string | null | undefined; + +function parseString(value: RedisValue | null | undefined): string | null | undefined { + if (value == null) { + return value; + } + + if (Buffer.isBuffer(value)) { + return value.toString('utf-8'); + } + + return String(value); +} + +type SetOp = SetOp1 | SetOp2 | SetOp3 | SetArg | undefined; +type SetOp1 = SetOp2 | 'NX' | 'XX'; +type SetOp2 = SetOp3 | 'GET'; +type SetOp3 = 'EX' | 'PX' | 'EXAT' | 'PXAT' | 'KEEPTTL'; +type SetArg = RedisNumber | RedisCallback; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type AnyCallback = (...args: any[]) => any; + +class EventManager { + private readonly groups = new Map(); + + public emit(ev: string, ...args: unknown[]) { + this.groups.get(ev)?.emit(args); + } + + @bindThis + public once(ev: string, callback: AnyCallback) { + this.on(ev, (...args: unknown[]) => { + this.off(ev, callback); + callback(...args); + }); + } + + @bindThis + public on(ev: string, callback: AnyCallback) { + this.makeEvent(ev).add(callback); + } + + @bindThis + public off(ev: string, callback: AnyCallback) { + this.groups.get(ev)?.remove(callback); + } + + private makeEvent(ev: string): EventGroup { + let group = this.groups.get(ev); + if (!group) { + group = new EventGroup(ev); + this.groups.set(ev, group); + } + return group; + } + + @bindThis + public clear() { + for (const group of this.groups.values()) { + group.clear(); + } + + this.groups.clear(); + } + + @bindThis + public dispose() { + for (const group of this.groups.values()) { + group.dispose(); + } + + this.clear(); + } +} + +class EventGroup { + private readonly listeners = new Set(); + + constructor( + public readonly ev: string, + ) {} + + public add(listener: AnyCallback): void { + this.listeners.add(listener); + } + + public remove(listener: AnyCallback): void { + this.listeners.delete(listener); + } + + public emit(...args: unknown[]): void { + for (const listener of this.listeners) { + listener(...args); + } + } + + public clear() { + this.listeners.clear(); + } + + public dispose() { + this.clear(); + } +} diff --git a/packages/backend/test/misc/custom-assertions.ts b/packages/backend/test/misc/custom-assertions.ts new file mode 100644 index 0000000000..7118b247f6 --- /dev/null +++ b/packages/backend/test/misc/custom-assertions.ts @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { inspect } from 'node:util'; +import { isError } from '@/misc/is-error.js'; + +export function isNotNullish(value: T): NonNullable { + expect(value).not.toBe(undefined); + expect(value).not.toBe(null); + return value as NonNullable; +} + +export function throws(callback: SyncCallback): unknown; +export function throws(errorClass: TError, callback: SyncCallback): InstanceType; +export function throws(errorClassOrCallback: TError | (SyncCallback), callbackOrUndefined?: SyncCallback): InstanceType | unknown { + const callback = (callbackOrUndefined ?? errorClassOrCallback) as SyncCallback; + const errorClass = callbackOrUndefined !== undefined ? (errorClassOrCallback as TError) : undefined; + + let result: unknown = undefined; + + try { + result = callback(); + } catch (error) { + assertIsErrorOfType(errorClass, error); + return error as InstanceType; + } + + const callbackName = callback.name || 'callback'; + const errorName = errorClass?.name || 'Error'; + const resultSummary = inspect(result); + throw new Error(`assert.throws: expected ${callbackName} to throw ${errorName}, but instead it returned: ${resultSummary}`, { cause: result }); +} + +export async function throwsAsync(callback: AsyncCallback): Promise; +export async function throwsAsync(errorClass: TError, callback: AsyncCallback): Promise>; +export async function throwsAsync(errorClassOrCallback: TError | AsyncCallback, callbackOrUndefined?: AsyncCallback): Promise | unknown> { + const callback = (callbackOrUndefined ?? errorClassOrCallback) as AsyncCallback; + const errorClass = callbackOrUndefined !== undefined ? (errorClassOrCallback as TError) : undefined; + + let result: unknown = undefined; + + try { + result = await callback(); + } catch (error) { + assertIsErrorOfType(errorClass, error); + return error as InstanceType; + } + + const callbackName = callback.name || 'callback'; + const errorName = errorClass?.name || 'Error'; + const resultSummary = inspect(result); + throw new Error(`assert.throwsAsync: expected ${callbackName} to throw ${errorName}, but instead it returned: ${resultSummary}`, { cause: result }); +} + +export async function rejectsAsync(promise: AnyPromise): Promise; +export async function rejectsAsync(errorClass: TError, promise: AnyPromise): Promise>; +export async function rejectsAsync(errorClassOrPromise: TError | AnyPromise, promiseOrUndefined?: AnyPromise): Promise | unknown> { + const promise = (promiseOrUndefined ?? errorClassOrPromise) as AnyPromise; + const errorClass = promiseOrUndefined !== undefined ? (errorClassOrPromise as TError) : undefined; + + let result: unknown = undefined; + + try { + result = await promise; + } catch (error) { + assertIsErrorOfType(errorClass, error); + return error as InstanceType; + } + + const errorName = errorClass?.name || 'Error'; + const resultSummary = inspect(result); + throw new Error(`assert.rejectsAsync: expected promise to reject with ${errorName}, but instead it resolved with ${resultSummary}`, { cause: result }); +} + +function assertIsErrorOfType(errorClass: AnyConstructor | undefined, error: unknown): void { + if (errorClass === Error) { + expect(error).toBeDefined(); + expect(isError(error)).toBe(true); + } else if (errorClass !== undefined) { + expect(error).toBeDefined(); + expect(error).toBeInstanceOf(errorClass); + } +} + +type AnyConstructor = abstract new(...args: unknown[]) => unknown; +type AnyPromise = Promise; +type SyncCallback = () => unknown; +type AsyncCallback = () => Promise; diff --git a/packages/backend/test/misc/mock-resolver.ts b/packages/backend/test/misc/mock-resolver.ts index 34241d13cb..3bb71a1471 100644 --- a/packages/backend/test/misc/mock-resolver.ts +++ b/packages/backend/test/misc/mock-resolver.ts @@ -3,13 +3,15 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { Inject } from '@nestjs/common'; +import { MockConsole } from './MockConsole.js'; +import { MockEnvService } from './MockEnvService.js'; import type { Config } from '@/config.js'; import type { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import type { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import type { ApRequestService } from '@/core/activitypub/ApRequestService.js'; import type { IObject, IObjectWithId } from '@/core/activitypub/type.js'; import type { HttpRequestService } from '@/core/HttpRequestService.js'; -import type { LoggerService } from '@/core/LoggerService.js'; import type { UtilityService } from '@/core/UtilityService.js'; import type { FollowRequestsRepository, @@ -21,11 +23,15 @@ import type { } from '@/models/_.js'; import type { CacheService } from '@/core/CacheService.js'; import { ApLogService } from '@/core/ApLogService.js'; +import { LoggerService } from '@/core/LoggerService.js'; import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { fromTuple } from '@/misc/from-tuple.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { bindThis } from '@/decorators.js'; import { Resolver } from '@/core/activitypub/ApResolverService.js'; +import { DI } from '@/di-symbols.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { NativeTimeService } from '@/global/TimeService.js'; type MockResponse = { type: string; @@ -36,25 +42,59 @@ export class MockResolver extends Resolver { #responseMap = new Map(); #remoteGetTrials: string[] = []; - constructor(loggerService: LoggerService) { + constructor( + @Inject(DI.config) + config?: Config, + + @Inject(DI.meta) + meta?: MiMeta, + + @Inject(DI.usersRepository) + usersRepository?: UsersRepository, + + @Inject(DI.notesRepository) + notesRepository?: NotesRepository, + + @Inject(DI.pollsRepository) + pollsRepository?: PollsRepository, + + @Inject(DI.noteReactionsRepository) + noteReactionsRepository?: NoteReactionsRepository, + + @Inject(DI.followRequestsRepository) + followRequestsRepository?: FollowRequestsRepository, + + utilityService?: UtilityService, + systemAccountService?: SystemAccountService, + apRequestService?: ApRequestService, + httpRequestService?: HttpRequestService, + apRendererService?: ApRendererService, + apDbResolverService?: ApDbResolverService, + loggerService?: LoggerService, + apLogService?: ApLogService, + apUtilityService?: ApUtilityService, + cacheService?: CacheService, + recursionLimit?: number, + ) { super( - {} as Config, - {} as MiMeta, - {} as UsersRepository, - {} as NotesRepository, - {} as PollsRepository, - {} as NoteReactionsRepository, - {} as FollowRequestsRepository, - {} as UtilityService, - {} as SystemAccountService, - {} as ApRequestService, - {} as HttpRequestService, - {} as ApRendererService, - {} as ApDbResolverService, - loggerService, - {} as ApLogService, - {} as ApUtilityService, - {} as CacheService, + config ?? {} as Config, + meta ?? {} as MiMeta, + usersRepository ?? {} as UsersRepository, + notesRepository ?? {} as NotesRepository, + pollsRepository ?? {} as PollsRepository, + noteReactionsRepository ?? {} as NoteReactionsRepository, + followRequestsRepository ?? {} as FollowRequestsRepository, + utilityService ?? {} as UtilityService, + systemAccountService ?? {} as SystemAccountService, + apRequestService ?? {} as ApRequestService, + httpRequestService ?? {} as HttpRequestService, + apRendererService ?? {} as ApRendererService, + apDbResolverService ?? {} as ApDbResolverService, + loggerService ?? new LoggerService(new MockConsole(), new NativeTimeService(), new MockEnvService()), + apLogService ?? {} as ApLogService, + apUtilityService ?? {} as ApUtilityService, + cacheService ?? {} as CacheService, + recursionLimit, ); } @@ -66,6 +106,7 @@ export class MockResolver extends Resolver { } public clear(): void { + this.history.clear(); this.#responseMap.clear(); this.#remoteGetTrials.length = 0; } @@ -81,6 +122,15 @@ export class MockResolver extends Resolver { value = fromTuple(value); if (typeof value !== 'string') return value; + // Check history - copied from Resolver._resolve + if (this.history.has(value)) { + throw new IdentifiableError('0dc86cf6-7cd6-4e56-b1e6-5903d62d7ea5', `failed to resolve ${value}: recursive resolution blocked`); + } + if (this.history.size > this.recursionLimit) { + throw new IdentifiableError('d592da9f-822f-4d91-83d7-4ceefabcf3d2', `failed to resolve ${value}: hit recursion limit`); + } + this.history.add(value); + this.#remoteGetTrials.push(value); const r = this.#responseMap.get(value); diff --git a/packages/backend/test/misc/noOpCaches.ts b/packages/backend/test/misc/noOpCaches.ts deleted file mode 100644 index 208d51f23a..0000000000 --- a/packages/backend/test/misc/noOpCaches.ts +++ /dev/null @@ -1,193 +0,0 @@ -/* - * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import * as Redis from 'ioredis'; -import { Inject } from '@nestjs/common'; -import { FakeInternalEventService } from './FakeInternalEventService.js'; -import type { BlockingsRepository, FollowingsRepository, MiUser, MutingsRepository, NoteThreadMutingsRepository, RenoteMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; -import type { MiLocalUser } from '@/models/User.js'; -import { MemoryKVCache, MemorySingleCache, RedisKVCache, RedisSingleCache } from '@/misc/cache.js'; -import { QuantumKVCache, QuantumKVOpts } from '@/misc/QuantumKVCache.js'; -import { CacheService, FollowStats } from '@/core/CacheService.js'; -import { DI } from '@/di-symbols.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; - -export function noOpRedis() { - return { - set: () => Promise.resolve(), - get: () => Promise.resolve(null), - del: () => Promise.resolve(), - on: () => {}, - off: () => {}, - } as unknown as Redis.Redis; -} - -export class NoOpCacheService extends CacheService { - public readonly fakeRedis: { - [K in keyof Redis.Redis]: Redis.Redis[K]; - }; - public readonly fakeInternalEventService: FakeInternalEventService; - - constructor( - @Inject(DI.usersRepository) - usersRepository: UsersRepository, - - @Inject(DI.userProfilesRepository) - userProfilesRepository: UserProfilesRepository, - - @Inject(DI.mutingsRepository) - mutingsRepository: MutingsRepository, - - @Inject(DI.blockingsRepository) - blockingsRepository: BlockingsRepository, - - @Inject(DI.renoteMutingsRepository) - renoteMutingsRepository: RenoteMutingsRepository, - - @Inject(DI.followingsRepository) - followingsRepository: FollowingsRepository, - - @Inject(DI.noteThreadMutingsRepository) - noteThreadMutingsRepository: NoteThreadMutingsRepository, - - @Inject(UserEntityService) - userEntityService: UserEntityService, - ) { - const fakeRedis = noOpRedis(); - const fakeInternalEventService = new FakeInternalEventService(); - - super( - fakeRedis, - fakeRedis, - usersRepository, - userProfilesRepository, - mutingsRepository, - blockingsRepository, - renoteMutingsRepository, - followingsRepository, - noteThreadMutingsRepository, - userEntityService, - fakeInternalEventService, - ); - - this.fakeRedis = fakeRedis; - this.fakeInternalEventService = fakeInternalEventService; - - // Override caches - this.userByIdCache = new NoOpMemoryKVCache(); - this.localUserByNativeTokenCache = new NoOpMemoryKVCache(); - this.localUserByIdCache = new NoOpMemoryKVCache(); - this.uriPersonCache = new NoOpMemoryKVCache(); - this.userProfileCache = NoOpQuantumKVCache.copy(this.userProfileCache, fakeInternalEventService); - this.userMutingsCache = NoOpQuantumKVCache.copy(this.userMutingsCache, fakeInternalEventService); - this.userBlockingCache = NoOpQuantumKVCache.copy(this.userBlockingCache, fakeInternalEventService); - this.userBlockedCache = NoOpQuantumKVCache.copy(this.userBlockedCache, fakeInternalEventService); - this.renoteMutingsCache = NoOpQuantumKVCache.copy(this.renoteMutingsCache, fakeInternalEventService); - this.threadMutingsCache = NoOpQuantumKVCache.copy(this.threadMutingsCache, fakeInternalEventService); - this.noteMutingsCache = NoOpQuantumKVCache.copy(this.noteMutingsCache, fakeInternalEventService); - this.userFollowingsCache = NoOpQuantumKVCache.copy(this.userFollowingsCache, fakeInternalEventService); - this.userFollowersCache = NoOpQuantumKVCache.copy(this.userFollowersCache, fakeInternalEventService); - this.hibernatedUserCache = NoOpQuantumKVCache.copy(this.hibernatedUserCache, fakeInternalEventService); - this.userFollowStatsCache = new NoOpMemoryKVCache(); - this.translationsCache = NoOpRedisKVCache.copy(this.translationsCache, fakeRedis); - } -} - -export class NoOpMemoryKVCache extends MemoryKVCache { - constructor() { - super(-1); - } -} - -export class NoOpMemorySingleCache extends MemorySingleCache { - constructor() { - super(-1); - } -} - -export class NoOpRedisKVCache extends RedisKVCache { - constructor(opts?: { - redis?: Redis.Redis; - fetcher?: RedisKVCache['fetcher']; - toRedisConverter?: RedisKVCache['toRedisConverter']; - fromRedisConverter?: RedisKVCache['fromRedisConverter']; - }) { - super( - opts?.redis ?? noOpRedis(), - 'no-op', - { - lifetime: -1, - memoryCacheLifetime: -1, - fetcher: opts?.fetcher, - toRedisConverter: opts?.toRedisConverter, - fromRedisConverter: opts?.fromRedisConverter, - }, - ); - } - - public static copy(cache: RedisKVCache, redis?: Redis.Redis): NoOpRedisKVCache { - return new NoOpRedisKVCache({ - redis, - fetcher: cache.fetcher, - toRedisConverter: cache.toRedisConverter, - fromRedisConverter: cache.fromRedisConverter, - }); - } -} - -export class NoOpRedisSingleCache extends RedisSingleCache { - constructor(opts?: { - redis?: Redis.Redis; - fetcher?: RedisSingleCache['fetcher']; - toRedisConverter?: RedisSingleCache['toRedisConverter']; - fromRedisConverter?: RedisSingleCache['fromRedisConverter']; - }) { - super( - opts?.redis ?? noOpRedis(), - 'no-op', - { - lifetime: -1, - memoryCacheLifetime: -1, - fetcher: opts?.fetcher, - toRedisConverter: opts?.toRedisConverter, - fromRedisConverter: opts?.fromRedisConverter, - }, - ); - } - - public static copy(cache: RedisSingleCache, redis?: Redis.Redis): NoOpRedisSingleCache { - return new NoOpRedisSingleCache({ - redis, - fetcher: cache.fetcher, - toRedisConverter: cache.toRedisConverter, - fromRedisConverter: cache.fromRedisConverter, - }); - } -} - -export class NoOpQuantumKVCache extends QuantumKVCache { - constructor(opts: Omit, 'lifetime'> & { - internalEventService?: InternalEventService, - }) { - super( - opts.internalEventService ?? new FakeInternalEventService(), - 'no-op', - { - ...opts, - lifetime: -1, - }, - ); - } - - public static copy(cache: QuantumKVCache, internalEventService?: InternalEventService): NoOpQuantumKVCache { - return new NoOpQuantumKVCache({ - internalEventService, - fetcher: cache.fetcher, - bulkFetcher: cache.bulkFetcher, - onChanged: cache.onChanged, - }); - } -} diff --git a/packages/backend/test/tsconfig.json b/packages/backend/test/tsconfig.json index f3b6a5108d..1591bd6dcb 100644 --- a/packages/backend/test/tsconfig.json +++ b/packages/backend/test/tsconfig.json @@ -1,43 +1,20 @@ { + "extends": "../../shared/tsconfig.node.jsonc", "compilerOptions": { - "allowJs": true, - "noEmitOnError": false, - "noImplicitAny": true, - "noImplicitReturns": true, - "noUnusedParameters": false, - "noUnusedLocals": false, - "noFallthroughCasesInSwitch": true, - "declaration": false, - "sourceMap": true, - "target": "ES2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "allowSyntheticDefaultImports": true, - "removeComments": false, - "noLib": false, - "strict": true, - "strictNullChecks": true, + // Checking + "types": ["jest", "node"], + "verbatimModuleSyntax": false, + "noImplicitOverride": false, + "noImplicitAny": false, + "strictFunctionTypes": false, "strictPropertyInitialization": false, - "skipLibCheck": true, - "experimentalDecorators": true, - "emitDecoratorMetadata": true, - "resolveJsonModule": true, - "isolatedModules": true, - "incremental": true, - "baseUrl": "./", "paths": { "@/*": ["../src/*"] }, - "typeRoots": [ - "../node_modules/@types", - "../src/@types" - ], - "lib": [ - "esnext" - ], - "types": ["jest", "node"] + + // Output + "outDir": "./built" }, - "compileOnSave": false, "include": [ "./**/*.ts", "../src/**/*.test.ts", diff --git a/packages/backend/test/unit/AbuseReportNotificationService.ts b/packages/backend/test/unit/AbuseReportNotificationService.ts index a67cb3664a..7095fb3445 100644 --- a/packages/backend/test/unit/AbuseReportNotificationService.ts +++ b/packages/backend/test/unit/AbuseReportNotificationService.ts @@ -29,6 +29,7 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import { RecipientMethod } from '@/models/AbuseReportNotificationRecipient.js'; import { SystemWebhookService } from '@/core/SystemWebhookService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { CoreModule } from '@/core/CoreModule.js'; process.env.NODE_ENV = 'test'; @@ -114,41 +115,25 @@ describe('AbuseReportNotificationService', () => { .createTestingModule({ imports: [ GlobalModule, - ], - providers: [ - AbuseReportNotificationService, - IdService, - { - provide: RoleService, useFactory: () => ({ getModeratorIds: jest.fn() }), - }, - { - provide: SystemWebhookService, useFactory: () => ({ enqueueSystemWebhook: jest.fn() }), - }, - { - provide: UserEntityService, useFactory: () => ({ - pack: (v: any) => Promise.resolve(v), - packMany: (v: any) => Promise.resolve(v), - }), - }, - { - provide: EmailService, useFactory: () => ({ sendEmail: jest.fn() }), - }, - { - provide: MetaService, useFactory: () => ({ fetch: jest.fn() }), - }, - { - provide: ModerationLogService, useFactory: () => ({ log: () => Promise.resolve() }), - }, - { - provide: GlobalEventService, useFactory: () => ({ publishAdminStream: jest.fn() }), - }, - { - provide: DI.meta, useFactory: () => meta, - }, + CoreModule, ], }) + .overrideProvider(RoleService).useValue({ getModeratorIds: jest.fn() }) + .overrideProvider(SystemWebhookService).useValue({ enqueueSystemWebhook: jest.fn() }) + .overrideProvider(UserEntityService).useValue({ + pack: (v: any) => Promise.resolve(v), + packMany: (v: any) => Promise.resolve(v), + }) + .overrideProvider(EmailService).useValue({ sendEmail: jest.fn() }) + .overrideProvider(MetaService).useValue({ fetch: jest.fn() }) + .overrideProvider(ModerationLogService).useValue({ log: () => Promise.resolve() }) + .overrideProvider(GlobalEventService).useValue({ publishAdminStream: jest.fn() }) + .overrideProvider(DI.meta).useValue(meta) .compile(); + await app.init(); + app.enableShutdownHooks(); + usersRepository = app.get(DI.usersRepository); userProfilesRepository = app.get(DI.userProfilesRepository); systemWebhooksRepository = app.get(DI.systemWebhooksRepository); @@ -159,8 +144,10 @@ describe('AbuseReportNotificationService', () => { roleService = app.get(RoleService) as jest.Mocked; emailService = app.get(EmailService) as jest.Mocked; webhookService = app.get(SystemWebhookService) as jest.Mocked; + }); - app.enableShutdownHooks(); + afterAll(async () => { + await app.close(); }); beforeEach(async () => { @@ -179,14 +166,10 @@ describe('AbuseReportNotificationService', () => { emailService.sendEmail.mockClear(); webhookService.enqueueSystemWebhook.mockClear(); - await usersRepository.delete({}); - await userProfilesRepository.delete({}); - await systemWebhooksRepository.delete({}); - await abuseReportNotificationRecipientRepository.delete({}); - }); - - afterAll(async () => { - await app.close(); + await usersRepository.deleteAll(); + await userProfilesRepository.deleteAll(); + await systemWebhooksRepository.deleteAll(); + await abuseReportNotificationRecipientRepository.deleteAll(); }); // -------------------------------------------------------------------------------------- diff --git a/packages/backend/test/unit/AnnouncementService.ts b/packages/backend/test/unit/AnnouncementService.ts index ab3b6961c0..1fef886b80 100644 --- a/packages/backend/test/unit/AnnouncementService.ts +++ b/packages/backend/test/unit/AnnouncementService.ts @@ -8,12 +8,13 @@ process.env.NODE_ENV = 'test'; import { jest } from '@jest/globals'; import { ModuleMocker } from 'jest-mock'; import { Test } from '@nestjs/testing'; -import { NoOpCacheService } from '../misc/noOpCaches.js'; -import { FakeInternalEventService } from '../misc/FakeInternalEventService.js'; +import { FakeCacheManagementService } from '../misc/FakeCacheManagementService.js'; +import { MockInternalEventService } from '../misc/MockInternalEventService.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { AnnouncementService } from '@/core/AnnouncementService.js'; import { AnnouncementEntityService } from '@/core/entities/AnnouncementEntityService.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; import type { AnnouncementReadsRepository, AnnouncementsRepository, @@ -23,14 +24,14 @@ import type { } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { genAidx } from '@/misc/id/aidx.js'; -import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { RoleService } from '@/core/RoleService.js'; +import { CoreModule } from '@/core/CoreModule.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; import type { TestingModule } from '@nestjs/testing'; -import type { MockFunctionMetadata } from 'jest-mock'; +import type { MockMetadata } from 'jest-mock'; const moduleMocker = new ModuleMocker(global); @@ -42,7 +43,7 @@ describe('AnnouncementService', () => { let announcementReadsRepository: AnnouncementReadsRepository; let globalEventService: jest.Mocked; let moderationLogService: jest.Mocked; - let roleService: jest.Mocked; + let cacheManagementService: CacheManagementService; function createUser(data: Partial = {}) { const un = secureRndstr(16); @@ -66,25 +67,16 @@ describe('AnnouncementService', () => { .then(x => announcementsRepository.findOneByOrFail(x.identifiers[0])); } - beforeEach(async () => { + beforeAll(async () => { app = await Test.createTestingModule({ imports: [ GlobalModule, - ], - providers: [ - AnnouncementService, - AnnouncementEntityService, - CacheService, - IdService, - InternalEventService, - GlobalEventService, - ModerationLogService, - RoleService, + CoreModule, ], }) .useMocker((token) => { if (typeof token === 'function') { - const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata; + const mockMetadata = moduleMocker.getMetadata(token) as MockMetadata; const Mock = moduleMocker.generateFromMetadata(mockMetadata); return new Mock(); } @@ -99,30 +91,38 @@ describe('AnnouncementService', () => { .overrideProvider(RoleService).useValue({ getUserRoles: jest.fn((_) => []), }) - .overrideProvider(InternalEventService).useClass(FakeInternalEventService) - .overrideProvider(CacheService).useClass(NoOpCacheService) + // TODO should we remove this now that cache is cleared? + .overrideProvider(InternalEventService).useClass(MockInternalEventService) + .overrideProvider(CacheManagementService).useClass(FakeCacheManagementService) .compile(); + await app.init(); app.enableShutdownHooks(); + }); + afterAll(async () => { + await app.close(); + }); + + beforeEach(() => { announcementService = app.get(AnnouncementService); usersRepository = app.get(DI.usersRepository); announcementsRepository = app.get(DI.announcementsRepository); announcementReadsRepository = app.get(DI.announcementReadsRepository); globalEventService = app.get(GlobalEventService) as jest.Mocked; moderationLogService = app.get(ModerationLogService) as jest.Mocked; - roleService = app.get(RoleService) as jest.Mocked; + cacheManagementService = app.get(CacheManagementService); }); afterEach(async () => { - await Promise.all([ - app.get(DI.metasRepository).delete({}), - usersRepository.delete({}), - announcementsRepository.delete({}), - announcementReadsRepository.delete({}), - ]); - - await app.close(); + await app.get(DI.metasRepository).deleteAll(); + await usersRepository.deleteAll(); + await announcementsRepository.deleteAll(); + await announcementReadsRepository.deleteAll(); + moderationLogService.log.mockReset(); + globalEventService.publishMainStream.mockReset(); + globalEventService.publishBroadcastStream.mockReset(); + cacheManagementService.clear(); }); describe('getUnreadAnnouncements', () => { diff --git a/packages/backend/test/unit/ApMfmService.ts b/packages/backend/test/unit/ApMfmService.ts index e81a321c9b..7bffd2dea7 100644 --- a/packages/backend/test/unit/ApMfmService.ts +++ b/packages/backend/test/unit/ApMfmService.ts @@ -4,21 +4,21 @@ */ import * as assert from 'assert'; -import { Test } from '@nestjs/testing'; - -import { CoreModule } from '@/core/CoreModule.js'; +import type { Config } from '@/config.js'; import { ApMfmService } from '@/core/activitypub/ApMfmService.js'; -import { GlobalModule } from '@/GlobalModule.js'; -import { MiNote } from '@/models/Note.js'; +import { MfmService } from '@/core/MfmService.js'; describe('ApMfmService', () => { + let config: Config; + let mfmService: MfmService; let apMfmService: ApMfmService; - beforeAll(async () => { - const app = await Test.createTestingModule({ - imports: [GlobalModule, CoreModule], - }).compile(); - apMfmService = app.get(ApMfmService); + beforeEach(() => { + config = { + url: 'http://misskey.local', + } as unknown as Config; + mfmService = new MfmService(config); + apMfmService = new ApMfmService(mfmService); }); describe('getNoteHtml', () => { diff --git a/packages/backend/test/unit/CaptchaService.ts b/packages/backend/test/unit/CaptchaService.ts index 94a743e6b8..1a2cc88ea8 100644 --- a/packages/backend/test/unit/CaptchaService.ts +++ b/packages/backend/test/unit/CaptchaService.ts @@ -8,8 +8,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { Response } from 'node-fetch'; import { CaptchaError, - CaptchaErrorCode, - captchaErrorCodes, CaptchaSaveResult, CaptchaService, } from '@/core/CaptchaService.js'; @@ -18,6 +16,8 @@ import { HttpRequestService } from '@/core/HttpRequestService.js'; import { MetaService } from '@/core/MetaService.js'; import { MiMeta } from '@/models/Meta.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { CoreModule } from '@/core/CoreModule.js'; +import { captchaErrorCodes, type CaptchaErrorCode } from '@/misc/captcha-error.js'; describe('CaptchaService', () => { let app: TestingModule; @@ -29,22 +29,14 @@ describe('CaptchaService', () => { app = await Test.createTestingModule({ imports: [ GlobalModule, + CoreModule, ], - providers: [ - CaptchaService, - LoggerService, - { - provide: HttpRequestService, useFactory: () => ({ send: jest.fn() }), - }, - { - provide: MetaService, useFactory: () => ({ - fetch: jest.fn(), - update: jest.fn(), - }), - }, - ], - }).compile(); + }) + .overrideProvider(HttpRequestService).useValue({ send: jest.fn() }) + .overrideProvider(MetaService).useValue({ fetch: jest.fn(), update: jest.fn() }) + .compile(); + await app.init(); app.enableShutdownHooks(); service = app.get(CaptchaService); @@ -463,7 +455,7 @@ describe('CaptchaService', () => { if (!res.success) { expect(res.error.code).toBe(code); } - expect(metaService.update).not.toBeCalled(); + expect(metaService.update).not.toHaveBeenCalled(); } describe('invalidParameters', () => { diff --git a/packages/backend/test/unit/CustomEmojiService.ts b/packages/backend/test/unit/CustomEmojiService.ts index 8c3dac69e8..01b0d09f58 100644 --- a/packages/backend/test/unit/CustomEmojiService.ts +++ b/packages/backend/test/unit/CustomEmojiService.ts @@ -5,6 +5,7 @@ import { afterEach, beforeAll, describe, test } from '@jest/globals'; import { Test, TestingModule } from '@nestjs/testing'; +import { DataSource } from 'typeorm'; import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; @@ -17,7 +18,6 @@ import { EmojisRepository } from '@/models/_.js'; import { MiEmoji } from '@/models/Emoji.js'; import { CoreModule } from '@/core/CoreModule.js'; import { DriveService } from '@/core//DriveService.js'; -import { DataSource } from 'typeorm'; describe('CustomEmojiService', () => { let app: TestingModule; @@ -33,23 +33,20 @@ describe('CustomEmojiService', () => { GlobalModule, CoreModule, ], - providers: [ - CustomEmojiService, - UtilityService, - IdService, - EmojiEntityService, - ModerationLogService, - GlobalEventService, - DriveService, - ], }) .compile(); + + await app.init(); app.enableShutdownHooks(); service = app.get(CustomEmojiService); emojisRepository = app.get(DI.emojisRepository); idService = app.get(IdService); - await app.get(DI.db).query("set session time zone 'UTC'"); + await app.get(DI.db).query('set session time zone \'UTC\''); + }); + + afterAll(async () => { + await app.close(); }); describe('fetchEmojis', () => { @@ -92,7 +89,7 @@ describe('CustomEmojiService', () => { } afterEach(async () => { - await emojisRepository.delete({}); + await emojisRepository.deleteAll(); }); describe('単独', () => { diff --git a/packages/backend/test/unit/DriveService.ts b/packages/backend/test/unit/DriveService.ts index 964c65ccaa..64babcb1a3 100644 --- a/packages/backend/test/unit/DriveService.ts +++ b/packages/backend/test/unit/DriveService.ts @@ -27,20 +27,22 @@ describe('DriveService', () => { beforeAll(async () => { app = await Test.createTestingModule({ imports: [GlobalModule, CoreModule], - providers: [DriveService], }).compile(); - app.enableShutdownHooks(); - driveService = app.get(DriveService); - }); - beforeEach(async () => { - s3Mock.reset(); + await app.init(); + app.enableShutdownHooks(); + + driveService = app.get(DriveService); }); afterAll(async () => { await app.close(); }); + beforeEach(async () => { + s3Mock.reset(); + }); + describe('Object storage', () => { test('delete a file', async () => { s3Mock.on(DeleteObjectCommand) @@ -53,7 +55,7 @@ describe('DriveService', () => { s3Mock.on(DeleteObjectCommand) .rejects(new InvalidObjectState({ $metadata: {}, message: '' })); - await expect(driveService.deleteObjectStorageFile('unexpected')).rejects.toThrowError(Error); + await expect(driveService.deleteObjectStorageFile('unexpected')).rejects.toBeInstanceOf(Error); }); test('delete a file with no valid key', async () => { diff --git a/packages/backend/test/unit/FetchInstanceMetadataService.ts b/packages/backend/test/unit/FetchInstanceMetadataService.ts index 812ee38703..aec9313377 100644 --- a/packages/backend/test/unit/FetchInstanceMetadataService.ts +++ b/packages/backend/test/unit/FetchInstanceMetadataService.ts @@ -7,8 +7,10 @@ process.env.NODE_ENV = 'test'; import { jest } from '@jest/globals'; import { Test } from '@nestjs/testing'; -import { Redis } from 'ioredis'; +import { GodOfTimeService } from '../misc/GodOfTimeService.js'; +import { MockRedis } from '../misc/MockRedis.js'; import type { TestingModule } from '@nestjs/testing'; +import type { InstancesRepository } from '@/models/_.js'; import { GlobalModule } from '@/GlobalModule.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; @@ -16,67 +18,69 @@ import { HttpRequestService } from '@/core/HttpRequestService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { IdService } from '@/core/IdService.js'; -import { EnvService } from '@/core/EnvService.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; +import { CoreModule } from '@/core/CoreModule.js'; import { DI } from '@/di-symbols.js'; - -function mockRedis() { - const hash = {} as any; - const set = jest.fn((key: string, value) => { - const ret = hash[key]; - hash[key] = value; - return ret; - }); - return set; -} +import { TimeService } from '@/global/TimeService.js'; describe('FetchInstanceMetadataService', () => { let app: TestingModule; - let fetchInstanceMetadataService: jest.Mocked; + let fetchInstanceMetadataService: FetchInstanceMetadataService; let federatedInstanceService: jest.Mocked; let httpRequestService: jest.Mocked; - let redisClient: jest.Mocked; + let redisClient: MockRedis; + let instancesRepository: InstancesRepository; + let cacheManagementService: CacheManagementService; + let timeService: GodOfTimeService; - beforeEach(async () => { + beforeAll(async () => { app = await Test .createTestingModule({ imports: [ GlobalModule, - ], - providers: [ - FetchInstanceMetadataService, - LoggerService, - UtilityService, - IdService, - EnvService, + CoreModule, ], }) - .useMocker((token) => { - if (token === HttpRequestService) { - return { getJson: jest.fn(), getHtml: jest.fn(), send: jest.fn() }; - } else if (token === FederatedInstanceService) { - return { fetchOrRegister: jest.fn() }; - } else if (token === DI.redis) { - return mockRedis; - } - return null; - }) + .overrideProvider(TimeService).useClass(GodOfTimeService) + .overrideProvider(HttpRequestService).useValue({ getJson: jest.fn(), getHtml: jest.fn(), send: jest.fn() }) + .overrideProvider(FederatedInstanceService).useValue({ fetchOrRegister: jest.fn() }) + .overrideProvider(DI.redis).useClass(MockRedis) + .overrideProvider(DI.redisForSub).useClass(MockRedis) + .overrideProvider(DI.redisForPub).useClass(MockRedis) .compile(); + await app.init(); app.enableShutdownHooks(); - fetchInstanceMetadataService = app.get(FetchInstanceMetadataService) as jest.Mocked; + fetchInstanceMetadataService = app.get(FetchInstanceMetadataService); federatedInstanceService = app.get(FederatedInstanceService) as jest.Mocked; - redisClient = app.get(DI.redis) as jest.Mocked; httpRequestService = app.get(HttpRequestService) as jest.Mocked; + instancesRepository = app.get(DI.instancesRepository); + cacheManagementService = app.get(CacheManagementService); + timeService = app.get(TimeService); + redisClient = app.get(DI.redis); }); - afterEach(async () => { + afterAll(async () => { await app.close(); }); + beforeEach(() => { + federatedInstanceService.fetchOrRegister.mockReset(); + httpRequestService.getJson.mockReset(); + httpRequestService.getHtml.mockReset(); + httpRequestService.send.mockReset(); + redisClient.mockReset(); + timeService.resetToNow(); + }); + + afterEach(async () => { + await instancesRepository.deleteAll(); + cacheManagementService.clear(); + }); + test('Lock and update', async () => { - redisClient.set = mockRedis(); - const now = Date.now(); + const now = timeService.now; federatedInstanceService.fetchOrRegister.mockResolvedValue({ infoUpdatedAt: { getTime: () => { return now - 10 * 1000 * 60 * 60 * 24; } } } as any); httpRequestService.getJson.mockImplementation(() => { throw Error(); }); const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock'); @@ -90,8 +94,7 @@ describe('FetchInstanceMetadataService', () => { }); test('Lock and don\'t update', async () => { - redisClient.set = mockRedis(); - const now = Date.now(); + const now = timeService.now; federatedInstanceService.fetchOrRegister.mockResolvedValue({ infoUpdatedAt: { getTime: () => now } } as any); httpRequestService.getJson.mockImplementation(() => { throw Error(); }); const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock'); @@ -105,8 +108,7 @@ describe('FetchInstanceMetadataService', () => { }); test('Do nothing when lock not acquired', async () => { - redisClient.set = mockRedis(); - const now = Date.now(); + const now = timeService.now; federatedInstanceService.fetchOrRegister.mockResolvedValue({ infoUpdatedAt: { getTime: () => now - 10 * 1000 * 60 * 60 * 24 } } as any); httpRequestService.getJson.mockImplementation(() => { throw Error(); }); await fetchInstanceMetadataService.tryLock('example.com'); @@ -121,8 +123,7 @@ describe('FetchInstanceMetadataService', () => { }); test('Do when lock not acquired but forced', async () => { - redisClient.set = mockRedis(); - const now = Date.now(); + const now = timeService.now; federatedInstanceService.fetchOrRegister.mockResolvedValue({ infoUpdatedAt: { getTime: () => now - 10 * 1000 * 60 * 60 * 24 } } as any); httpRequestService.getJson.mockImplementation(() => { throw Error(); }); await fetchInstanceMetadataService.tryLock('example.com'); diff --git a/packages/backend/test/unit/FileInfoService.ts b/packages/backend/test/unit/FileInfoService.ts index 89583e7304..6eae604bf6 100644 --- a/packages/backend/test/unit/FileInfoService.ts +++ b/packages/backend/test/unit/FileInfoService.ts @@ -11,12 +11,13 @@ import { dirname } from 'node:path'; import { ModuleMocker } from 'jest-mock'; import { Test } from '@nestjs/testing'; import { afterAll, beforeAll, describe, test } from '@jest/globals'; +import type { MockMetadata } from 'jest-mock'; import { GlobalModule } from '@/GlobalModule.js'; import { FileInfo, FileInfoService } from '@/core/FileInfoService.js'; //import { DI } from '@/di-symbols.js'; import { LoggerService } from '@/core/LoggerService.js'; import type { TestingModule } from '@nestjs/testing'; -import type { MockFunctionMetadata } from 'jest-mock'; +import { CoreModule } from '@/core/CoreModule.js'; const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); @@ -41,21 +42,19 @@ describe('FileInfoService', () => { app = await Test.createTestingModule({ imports: [ GlobalModule, - ], - providers: [ - LoggerService, - FileInfoService, + CoreModule, ], }) .useMocker((token) => { if (typeof token === 'function') { - const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata; + const mockMetadata = moduleMocker.getMetadata(token) as MockMetadata; const Mock = moduleMocker.generateFromMetadata(mockMetadata); return new Mock(); } }) .compile(); + await app.init(); app.enableShutdownHooks(); fileInfoService = app.get(FileInfoService); diff --git a/packages/backend/test/unit/FlashService.ts b/packages/backend/test/unit/FlashService.ts index f2d9832f50..7b9be5bfe9 100644 --- a/packages/backend/test/unit/FlashService.ts +++ b/packages/backend/test/unit/FlashService.ts @@ -9,7 +9,9 @@ import { FlashService } from '@/core/FlashService.js'; import { IdService } from '@/core/IdService.js'; import { FlashsRepository, MiFlash, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; import { GlobalModule } from '@/GlobalModule.js'; +import { CoreModule } from '@/core/CoreModule.js'; describe('FlashService', () => { let app: TestingModule; @@ -21,6 +23,7 @@ describe('FlashService', () => { let usersRepository: UsersRepository; let userProfilesRepository: UserProfilesRepository; let idService: IdService; + let cacheManagementService: CacheManagementService; // -------------------------------------------------------------------------------------- @@ -61,37 +64,41 @@ describe('FlashService', () => { // -------------------------------------------------------------------------------------- - beforeEach(async () => { + beforeAll(async () => { app = await Test.createTestingModule({ imports: [ GlobalModule, - ], - providers: [ - FlashService, - IdService, + CoreModule, ], }).compile(); + await app.init(); + app.enableShutdownHooks(); + service = app.get(FlashService); flashsRepository = app.get(DI.flashsRepository); usersRepository = app.get(DI.usersRepository); userProfilesRepository = app.get(DI.userProfilesRepository); idService = app.get(IdService); + cacheManagementService = app.get(CacheManagementService); + }); + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { root = await createUser({ username: 'root', usernameLower: 'root' }); alice = await createUser({ username: 'alice', usernameLower: 'alice' }); bob = await createUser({ username: 'bob', usernameLower: 'bob' }); }); afterEach(async () => { - await usersRepository.delete({}); - await userProfilesRepository.delete({}); - await flashsRepository.delete({}); - }); - - afterAll(async () => { - await app.close(); + await usersRepository.deleteAll(); + await userProfilesRepository.deleteAll(); + await flashsRepository.deleteAll(); + cacheManagementService.clear(); }); // -------------------------------------------------------------------------------------- diff --git a/packages/backend/test/unit/MetaService.ts b/packages/backend/test/unit/MetaService.ts index 056838e180..5c1996045b 100644 --- a/packages/backend/test/unit/MetaService.ts +++ b/packages/backend/test/unit/MetaService.ts @@ -27,6 +27,7 @@ describe('MetaService', () => { ], }).compile(); + await app.init(); app.enableShutdownHooks(); metaService = app.get(MetaService, { strict: false }); diff --git a/packages/backend/test/unit/MfmService.ts b/packages/backend/test/unit/MfmService.ts index ace67caf33..8366f8ed59 100644 --- a/packages/backend/test/unit/MfmService.ts +++ b/packages/backend/test/unit/MfmService.ts @@ -5,20 +5,18 @@ import * as assert from 'assert'; import * as mfm from 'mfm-js'; -import { Test } from '@nestjs/testing'; - -import { CoreModule } from '@/core/CoreModule.js'; +import type { Config } from '@/config.js'; import { MfmService } from '@/core/MfmService.js'; -import { GlobalModule } from '@/GlobalModule.js'; describe('MfmService', () => { + let config: Config; let mfmService: MfmService; - beforeAll(async () => { - const app = await Test.createTestingModule({ - imports: [GlobalModule, CoreModule], - }).compile(); - mfmService = app.get(MfmService); + beforeEach(() => { + config = { + url: 'https://example.com', + } as unknown as Config; + mfmService = new MfmService(config); }); describe('toHtml', () => { @@ -72,22 +70,22 @@ describe('MfmService', () => { }); describe('toMastoApiHtml', () => { - test('br', async () => { + test('br', () => { const input = 'foo\nbar\nbaz'; const output = '

foo
bar
baz

'; - assert.equal(await mfmService.toMastoApiHtml(mfm.parse(input)), output); + assert.equal(mfmService.toMastoApiHtml(mfm.parse(input)), output); }); - test('br alt', async () => { + test('br alt', () => { const input = 'foo\r\nbar\rbaz'; const output = '

foo
bar
baz

'; - assert.equal(await mfmService.toMastoApiHtml(mfm.parse(input)), output); + assert.equal(mfmService.toMastoApiHtml(mfm.parse(input)), output); }); - test('escape', async () => { + test('escape', () => { const input = '```\n

Hello, world!

\n```'; const output = '

<p>Hello, world!</p>

'; - assert.equal(await mfmService.toMastoApiHtml(mfm.parse(input)), output); + assert.equal(mfmService.toMastoApiHtml(mfm.parse(input)), output); }); test('ruby', async () => { @@ -168,7 +166,7 @@ describe('MfmService', () => { assert.deepStrictEqual(mfmService.fromHtml('

a Misskey(ミス キー) b c

'), 'a $[ruby $[group Misskey] ミス キー] b c'); assert.deepStrictEqual( mfmService.fromHtml('

a Misskey(ミスキー)Misskey(ミス キー)Misskey(ミスキー) b

'), - 'a $[ruby Misskey ミスキー]$[ruby $[group Misskey] ミス キー]$[ruby Misskey ミスキー] b' + 'a $[ruby Misskey ミスキー]$[ruby $[group Misskey] ミス キー]$[ruby Misskey ミスキー] b', ); }); @@ -187,7 +185,7 @@ describe('MfmService', () => { test('ruby', () => { assert.deepStrictEqual( mfmService.fromHtml(' some text (ignore me) and more'), - '$[ruby $[group some text ] ignore me]$[ruby $[group and ] more]' + '$[ruby $[group some text ] ignore me]$[ruby $[group and ] more]', ); }); }); diff --git a/packages/backend/test/unit/NoteCreateService.ts b/packages/backend/test/unit/NoteCreateService.ts index 8f241cf0c7..2869dcc5af 100644 --- a/packages/backend/test/unit/NoteCreateService.ts +++ b/packages/backend/test/unit/NoteCreateService.ts @@ -3,25 +3,12 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Test } from '@nestjs/testing'; - -import { CoreModule } from '@/core/CoreModule.js'; -import { NoteCreateService } from '@/core/NoteCreateService.js'; -import { GlobalModule } from '@/GlobalModule.js'; +import { isRenote, isQuote } from '@/core/NoteCreateService.js'; import { MiNote } from '@/models/Note.js'; import { IPoll } from '@/models/Poll.js'; import { MiDriveFile } from '@/models/DriveFile.js'; describe('NoteCreateService', () => { - let noteCreateService: NoteCreateService; - - beforeAll(async () => { - const app = await Test.createTestingModule({ - imports: [GlobalModule, CoreModule], - }).compile(); - noteCreateService = app.get(NoteCreateService); - }); - describe('is-renote', () => { const base: MiNote = { id: 'some-note-id', @@ -108,43 +95,45 @@ describe('NoteCreateService', () => { test('note without renote should not be Renote', () => { const note = { renote: null }; - expect(noteCreateService['isRenote'](note)).toBe(false); + expect(isRenote(note)).toBe(false); }); test('note with renote should be Renote and not be Quote', () => { const note = { renote: base }; - expect(noteCreateService['isRenote'](note)).toBe(true); - expect(noteCreateService['isQuote'](note)).toBe(false); + expect(isRenote(note)).toBe(true); + expect(isQuote(note)).toBe(false); }); test('note with renote and text should be Quote', () => { const note = { renote: base, text: 'some-text' }; - expect(noteCreateService['isRenote'](note)).toBe(true); - expect(noteCreateService['isQuote'](note)).toBe(true); + expect(isRenote(note)).toBe(true); + expect(isQuote(note)).toBe(true); }); test('note with renote and cw should be Quote', () => { const note = { renote: base, cw: 'some-cw' }; - expect(noteCreateService['isRenote'](note)).toBe(true); - expect(noteCreateService['isQuote'](note)).toBe(true); + expect(isRenote(note)).toBe(true); + expect(isQuote(note)).toBe(true); }); test('note with renote and reply should be Quote', () => { const note = { renote: base, reply: { ...base, id: 'another-note-id' } }; - expect(noteCreateService['isRenote'](note)).toBe(true); - expect(noteCreateService['isQuote'](note)).toBe(true); + expect(isRenote(note)).toBe(true); + expect(isQuote(note)).toBe(true); }); test('note with renote and poll should be Quote', () => { const note = { renote: base, poll }; - expect(noteCreateService['isRenote'](note)).toBe(true); - expect(noteCreateService['isQuote'](note)).toBe(true); + expect(isRenote(note)).toBe(true); + expect(isQuote(note)).toBe(true); }); test('note with renote and non-empty files should be Quote', () => { const note = { renote: base, files: [file] }; - expect(noteCreateService['isRenote'](note)).toBe(true); - expect(noteCreateService['isQuote'](note)).toBe(true); + expect(isRenote(note)).toBe(true); + expect(isQuote(note)).toBe(true); }); }); + + // TODO tests for isPureRenote }); diff --git a/packages/backend/test/unit/ReactionService.ts b/packages/backend/test/unit/ReactionService.ts index 1957f4544c..429b276540 100644 --- a/packages/backend/test/unit/ReactionService.ts +++ b/packages/backend/test/unit/ReactionService.ts @@ -4,131 +4,118 @@ */ import * as assert from 'assert'; -import { Test } from '@nestjs/testing'; - -import { CoreModule } from '@/core/CoreModule.js'; -import { ReactionService } from '@/core/ReactionService.js'; -import { GlobalModule } from '@/GlobalModule.js'; +import { normalize, convertLegacyReactions } from '@/core/ReactionService.js'; describe('ReactionService', () => { - let reactionService: ReactionService; - - beforeAll(async () => { - const app = await Test.createTestingModule({ - imports: [GlobalModule, CoreModule], - }).compile(); - reactionService = app.get(ReactionService); - }); - describe('normalize', () => { test('絵文字リアクションはそのまま', async () => { - assert.strictEqual(await reactionService.normalize('👍'), '👍'); - assert.strictEqual(await reactionService.normalize('🍅'), '🍅'); + assert.strictEqual(normalize('👍'), '👍'); + assert.strictEqual(normalize('🍅'), '🍅'); }); test('既存のリアクションは絵文字化する pudding', async () => { - assert.strictEqual(await reactionService.normalize('pudding'), '🍮'); + assert.strictEqual(normalize('pudding'), '🍮'); }); test('既存のリアクションは絵文字化する like', async () => { - assert.strictEqual(await reactionService.normalize('like'), '👍'); + assert.strictEqual(normalize('like'), '👍'); }); test('既存のリアクションは絵文字化する love', async () => { - assert.strictEqual(await reactionService.normalize('love'), '❤'); + assert.strictEqual(normalize('love'), '❤'); }); test('既存のリアクションは絵文字化する laugh', async () => { - assert.strictEqual(await reactionService.normalize('laugh'), '😆'); + assert.strictEqual(normalize('laugh'), '😆'); }); test('既存のリアクションは絵文字化する hmm', async () => { - assert.strictEqual(await reactionService.normalize('hmm'), '🤔'); + assert.strictEqual(normalize('hmm'), '🤔'); }); test('既存のリアクションは絵文字化する surprise', async () => { - assert.strictEqual(await reactionService.normalize('surprise'), '😮'); + assert.strictEqual(normalize('surprise'), '😮'); }); test('既存のリアクションは絵文字化する congrats', async () => { - assert.strictEqual(await reactionService.normalize('congrats'), '🎉'); + assert.strictEqual(normalize('congrats'), '🎉'); }); test('既存のリアクションは絵文字化する angry', async () => { - assert.strictEqual(await reactionService.normalize('angry'), '💢'); + assert.strictEqual(normalize('angry'), '💢'); }); test('既存のリアクションは絵文字化する confused', async () => { - assert.strictEqual(await reactionService.normalize('confused'), '😥'); + assert.strictEqual(normalize('confused'), '😥'); }); test('既存のリアクションは絵文字化する rip', async () => { - assert.strictEqual(await reactionService.normalize('rip'), '😇'); + assert.strictEqual(normalize('rip'), '😇'); }); test('既存のリアクションは絵文字化する star', async () => { - assert.strictEqual(await reactionService.normalize('star'), '⭐'); + assert.strictEqual(normalize('star'), '⭐'); }); test('異体字セレクタ除去', async () => { - assert.strictEqual(await reactionService.normalize('㊗️'), '㊗'); + assert.strictEqual(normalize('㊗️'), '㊗'); }); test('異体字セレクタ除去 必要なし', async () => { - assert.strictEqual(await reactionService.normalize('㊗'), '㊗'); + assert.strictEqual(normalize('㊗'), '㊗'); }); test('fallback - null', async () => { - assert.strictEqual(await reactionService.normalize(null), '❤'); + assert.strictEqual(normalize(null), '❤'); }); test('fallback - empty', async () => { - assert.strictEqual(await reactionService.normalize(''), '❤'); + assert.strictEqual(normalize(''), '❤'); }); test('fallback - unknown', async () => { - assert.strictEqual(await reactionService.normalize('unknown'), '❤'); + assert.strictEqual(normalize('unknown'), '❤'); }); }); describe('convertLegacyReactions', () => { test('空の入力に対しては何もしない', () => { const input = {}; - assert.deepStrictEqual(reactionService.convertLegacyReactions(input), input); + assert.deepStrictEqual(convertLegacyReactions(input), input); }); test('Unicode絵文字リアクションを変換してしまわない', () => { const input = { '👍': 1, '🍮': 2 }; - assert.deepStrictEqual(reactionService.convertLegacyReactions(input), input); + assert.deepStrictEqual(convertLegacyReactions(input), input); }); test('カスタム絵文字リアクションを変換してしまわない', () => { const input = { ':like@.:': 1, ':pudding@example.tld:': 2 }; - assert.deepStrictEqual(reactionService.convertLegacyReactions(input), input); + assert.deepStrictEqual(convertLegacyReactions(input), input); }); test('文字列によるレガシーなリアクションを変換する', () => { const input = { 'like': 1, 'pudding': 2 }; const output = { '👍': 1, '🍮': 2 }; - assert.deepStrictEqual(reactionService.convertLegacyReactions(input), output); + assert.deepStrictEqual(convertLegacyReactions(input), output); }); test('host部分が省略されたレガシーなカスタム絵文字リアクションを変換する', () => { const input = { ':custom_emoji:': 1 }; const output = { ':custom_emoji@.:': 1 }; - assert.deepStrictEqual(reactionService.convertLegacyReactions(input), output); + assert.deepStrictEqual(convertLegacyReactions(input), output); }); test('「0個のリアクション」情報を削除する', () => { const input = { 'angry': 0 }; const output = {}; - assert.deepStrictEqual(reactionService.convertLegacyReactions(input), output); + assert.deepStrictEqual(convertLegacyReactions(input), output); }); test('host部分の有無によりデコードすると同じ表記になるカスタム絵文字リアクションの個数情報を正しく足し合わせる', () => { const input = { ':custom_emoji:': 1, ':custom_emoji@.:': 2 }; const output = { ':custom_emoji@.:': 3 }; - assert.deepStrictEqual(reactionService.convertLegacyReactions(input), output); + assert.deepStrictEqual(convertLegacyReactions(input), output); }); }); }); diff --git a/packages/backend/test/unit/RelayService.ts b/packages/backend/test/unit/RelayService.ts index 074430dd31..7a9c56de65 100644 --- a/packages/backend/test/unit/RelayService.ts +++ b/packages/backend/test/unit/RelayService.ts @@ -9,7 +9,7 @@ import { jest } from '@jest/globals'; import { Test } from '@nestjs/testing'; import { ModuleMocker } from 'jest-mock'; import type { TestingModule } from '@nestjs/testing'; -import type { MockFunctionMetadata } from 'jest-mock'; +import type { MockMetadata } from 'jest-mock'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { IdService } from '@/core/IdService.js'; @@ -18,6 +18,7 @@ import { RelayService } from '@/core/RelayService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { CoreModule } from '@/core/CoreModule.js'; const moduleMocker = new ModuleMocker(global); @@ -30,28 +31,20 @@ describe('RelayService', () => { app = await Test.createTestingModule({ imports: [ GlobalModule, - ], - providers: [ - IdService, - ApRendererService, - RelayService, - UserEntityService, - SystemAccountService, - UtilityService, + CoreModule, ], }) .useMocker((token) => { - if (token === QueueService) { - return { deliver: jest.fn() }; - } if (typeof token === 'function') { - const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata; + const mockMetadata = moduleMocker.getMetadata(token) as MockMetadata; const Mock = moduleMocker.generateFromMetadata(mockMetadata); return new Mock(); } }) + .overrideProvider(QueueService).useValue({ deliver: jest.fn() }) .compile(); + await app.init(); app.enableShutdownHooks(); relayService = app.get(RelayService); @@ -62,6 +55,10 @@ describe('RelayService', () => { await app.close(); }); + afterEach(() => { + queueService.deliver.mockReset(); + }); + test('addRelay', async () => { const result = await relayService.addRelay('https://example.com'); diff --git a/packages/backend/test/unit/RoleService.ts b/packages/backend/test/unit/RoleService.ts index edeac6f9ce..75af74988d 100644 --- a/packages/backend/test/unit/RoleService.ts +++ b/packages/backend/test/unit/RoleService.ts @@ -5,17 +5,16 @@ process.env.NODE_ENV = 'test'; -import { setTimeout } from 'node:timers/promises'; import { jest } from '@jest/globals'; import { ModuleMocker } from 'jest-mock'; import { Test } from '@nestjs/testing'; -import * as lolex from '@sinonjs/fake-timers'; -import { NoOpCacheService } from '../misc/noOpCaches.js'; -import { FakeInternalEventService } from '../misc/FakeInternalEventService.js'; import type { TestingModule } from '@nestjs/testing'; -import type { MockFunctionMetadata } from 'jest-mock'; +import type { MockMetadata } from 'jest-mock'; +import { GodOfTimeService } from '../misc/GodOfTimeService.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { RoleService } from '@/core/RoleService.js'; +import { CoreModule } from '@/core/CoreModule.js'; import { InstancesRepository, MetasRepository, @@ -37,7 +36,8 @@ import { secureRndstr } from '@/misc/secure-rndstr.js'; import { NotificationService } from '@/core/NotificationService.js'; import { RoleCondFormulaValue } from '@/models/Role.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { TimeService } from '@/global/TimeService.js'; const moduleMocker = new ModuleMocker(global); @@ -51,7 +51,8 @@ describe('RoleService', () => { let meta: jest.Mocked; let metasRepository: MetasRepository; let notificationService: jest.Mocked; - let clock: lolex.InstalledClock; + let cacheManagementService: CacheManagementService; + let timeService: GodOfTimeService; async function createUser(data: Partial = {}) { if (data.host != null) { @@ -59,8 +60,8 @@ describe('RoleService', () => { .createQueryBuilder('instance') .insert() .values({ - id: genAidx(Date.now()), - firstRetrievedAt: new Date(), + id: genAidx(timeService.now), + firstRetrievedAt: timeService.date, host: data.host, }) .orIgnore() @@ -69,7 +70,7 @@ describe('RoleService', () => { const un = secureRndstr(16); const x = await usersRepository.insert({ - id: genAidx(Date.now()), + id: genAidx(timeService.now), username: un, usernameLower: un, ...data, @@ -85,9 +86,9 @@ describe('RoleService', () => { async function createRole(data: Partial = {}) { const x = await rolesRepository.insert({ - id: genAidx(Date.now()), - updatedAt: new Date(), - lastUsedAt: new Date(), + id: genAidx(timeService.now), + updatedAt: timeService.date, + lastUsedAt: timeService.date, name: '', description: '', ...data, @@ -105,8 +106,8 @@ describe('RoleService', () => { } async function assignRole(args: Partial) { - const id = genAidx(Date.now()); - const expiresAt = new Date(); + const id = genAidx(timeService.now); + const expiresAt = timeService.date; expiresAt.setDate(expiresAt.getDate() + 1); await roleAssignmentsRepository.insert({ @@ -119,51 +120,29 @@ describe('RoleService', () => { } function aidx() { - return genAidx(Date.now()); + return genAidx(timeService.now); } - beforeEach(async () => { - clock = lolex.install({ - now: new Date(), - shouldClearNativeTimers: true, - }); - + beforeAll(async () => { app = await Test.createTestingModule({ imports: [ GlobalModule, - ], - providers: [ - RoleService, - CacheService, - IdService, - GlobalEventService, - UserEntityService, - { - provide: NotificationService, - useFactory: () => ({ - createNotification: jest.fn(), - }), - }, - { - provide: NotificationService.name, - useExisting: NotificationService, - }, - MetaService, - InternalEventService, + CoreModule, ], }) .useMocker((token) => { if (typeof token === 'function') { - const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata; + const mockMetadata = moduleMocker.getMetadata(token) as MockMetadata; const Mock = moduleMocker.generateFromMetadata(mockMetadata); return new Mock(); } }) .overrideProvider(MetaService).useValue({ fetch: jest.fn() }) - .overrideProvider(InternalEventService).useClass(FakeInternalEventService) - .overrideProvider(CacheService).useClass(NoOpCacheService) + .overrideProvider(TimeService).useClass(GodOfTimeService) + .overrideProvider(NotificationService).useValue({ createNotification: jest.fn() }) .compile(); + await app.init(); app.enableShutdownHooks(); roleService = app.get(RoleService); @@ -172,24 +151,31 @@ describe('RoleService', () => { rolesRepository = app.get(DI.rolesRepository); roleAssignmentsRepository = app.get(DI.roleAssignmentsRepository); metasRepository = app.get(DI.metasRepository); + cacheManagementService = app.get(CacheManagementService); + + timeService = app.get(TimeService); + timeService.resetToNow(); meta = app.get(DI.meta) as jest.Mocked; notificationService = app.get(NotificationService) as jest.Mocked; + }); - await roleService.onModuleInit(); + afterAll(async () => { + await app.close(); + }); + + beforeEach(() => { + notificationService.createNotification.mockReset(); + timeService.resetToNow(); }); afterEach(async () => { - clock.uninstall(); + await roleAssignmentsRepository.deleteAll(); + await rolesRepository.deleteAll(); + await metasRepository.deleteAll(); + await usersRepository.deleteAll(); - await Promise.all([ - metasRepository.delete({}), - usersRepository.delete({}), - rolesRepository.delete({}), - roleAssignmentsRepository.delete({}), - ]); - - await app.close(); + cacheManagementService.clear(); }); describe('getUserPolicies', () => { @@ -282,25 +268,28 @@ describe('RoleService', () => { }, }, }); - await roleService.assign(user.id, role.id, new Date(Date.now() + (1000 * 60 * 60 * 24))); + await roleService.assign(user.id, role.id, new Date(timeService.now + (1000 * 60 * 60 * 24))); meta.policies = { canManageCustomEmojis: false, }; + // Condition 1: user has role immediately after assigning const result = await roleService.getUserPolicies(user.id); expect(result.canManageCustomEmojis).toBe(true); - clock.tick('25:00:00'); + timeService.now += 1000 * 60 * 60 * 25; // 25h + // Condition 2: user loses role within 1hr after expiration const resultAfter25h = await roleService.getUserPolicies(user.id); expect(resultAfter25h.canManageCustomEmojis).toBe(false); await roleService.assign(user.id, role.id); // ストリーミング経由で反映されるまでちょっと待つ - clock.uninstall(); - await setTimeout(100); + // Wait 100ms for the task queue to complete. + await new Promise(r => setTimeout(r, 100)); + // Condition 3: user regains role within 100ms after assigning again const resultAfter25hAgain = await roleService.getUserPolicies(user.id); expect(resultAfter25hAgain.canManageCustomEmojis).toBe(true); }); @@ -318,11 +307,11 @@ describe('RoleService', () => { await Promise.all([ assignRole({ userId: adminUser1.id, roleId: role1.id }), - assignRole({ userId: adminUser2.id, roleId: role1.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: adminUser2.id, roleId: role1.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: modeUser1.id, roleId: role2.id }), - assignRole({ userId: modeUser2.id, roleId: role2.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: modeUser2.id, roleId: role2.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: normalUser1.id, roleId: role3.id }), - assignRole({ userId: normalUser2.id, roleId: role3.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: normalUser2.id, roleId: role3.id, expiresAt: new Date(timeService.now - 1000) }), ]); const result = await roleService.getModeratorIds({ @@ -344,11 +333,11 @@ describe('RoleService', () => { await Promise.all([ assignRole({ userId: adminUser1.id, roleId: role1.id }), - assignRole({ userId: adminUser2.id, roleId: role1.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: adminUser2.id, roleId: role1.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: modeUser1.id, roleId: role2.id }), - assignRole({ userId: modeUser2.id, roleId: role2.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: modeUser2.id, roleId: role2.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: normalUser1.id, roleId: role3.id }), - assignRole({ userId: normalUser2.id, roleId: role3.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: normalUser2.id, roleId: role3.id, expiresAt: new Date(timeService.now - 1000) }), ]); const result = await roleService.getModeratorIds({ @@ -370,11 +359,11 @@ describe('RoleService', () => { await Promise.all([ assignRole({ userId: adminUser1.id, roleId: role1.id }), - assignRole({ userId: adminUser2.id, roleId: role1.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: adminUser2.id, roleId: role1.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: modeUser1.id, roleId: role2.id }), - assignRole({ userId: modeUser2.id, roleId: role2.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: modeUser2.id, roleId: role2.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: normalUser1.id, roleId: role3.id }), - assignRole({ userId: normalUser2.id, roleId: role3.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: normalUser2.id, roleId: role3.id, expiresAt: new Date(timeService.now - 1000) }), ]); const result = await roleService.getModeratorIds({ @@ -396,11 +385,11 @@ describe('RoleService', () => { await Promise.all([ assignRole({ userId: adminUser1.id, roleId: role1.id }), - assignRole({ userId: adminUser2.id, roleId: role1.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: adminUser2.id, roleId: role1.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: modeUser1.id, roleId: role2.id }), - assignRole({ userId: modeUser2.id, roleId: role2.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: modeUser2.id, roleId: role2.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: normalUser1.id, roleId: role3.id }), - assignRole({ userId: normalUser2.id, roleId: role3.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: normalUser2.id, roleId: role3.id, expiresAt: new Date(timeService.now - 1000) }), ]); const result = await roleService.getModeratorIds({ @@ -422,11 +411,11 @@ describe('RoleService', () => { await Promise.all([ assignRole({ userId: adminUser1.id, roleId: role1.id }), - assignRole({ userId: adminUser2.id, roleId: role1.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: adminUser2.id, roleId: role1.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: modeUser1.id, roleId: role2.id }), - assignRole({ userId: modeUser2.id, roleId: role2.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: modeUser2.id, roleId: role2.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: normalUser1.id, roleId: role3.id }), - assignRole({ userId: normalUser2.id, roleId: role3.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: normalUser2.id, roleId: role3.id, expiresAt: new Date(timeService.now - 1000) }), ]); const result = await roleService.getModeratorIds({ @@ -496,8 +485,8 @@ describe('RoleService', () => { await Promise.all([ assignRole({ userId: adminUser1.id, roleId: role1.id }), - assignRole({ userId: modeUser1.id, roleId: role2.id, expiresAt: new Date(Date.now() - 1000) }), - assignRole({ userId: rootUser.id, roleId: role2.id, expiresAt: new Date(Date.now() - 1000) }), + assignRole({ userId: modeUser1.id, roleId: role2.id, expiresAt: new Date(timeService.now - 1000) }), + assignRole({ userId: rootUser.id, roleId: role2.id, expiresAt: new Date(timeService.now - 1000) }), assignRole({ userId: normalUser1.id, roleId: role3.id }), ]); @@ -745,7 +734,7 @@ describe('RoleService', () => { }); test('ユーザが作成されてから指定期間経過した', async () => { - const base = new Date(); + const base = timeService.date; base.setMinutes(base.getMinutes() - 5); const d1 = new Date(base); @@ -778,7 +767,7 @@ describe('RoleService', () => { }); test('ユーザが作成されてから指定期間経っていない', async () => { - const base = new Date(); + const base = timeService.date; base.setMinutes(base.getMinutes() - 5); const d1 = new Date(base); @@ -941,8 +930,7 @@ describe('RoleService', () => { await roleService.assign(user.id, role.id); - clock.uninstall(); - await setTimeout(100); + timeService.now += 100; const assignments = await roleAssignmentsRepository.find({ where: { @@ -969,8 +957,7 @@ describe('RoleService', () => { await roleService.assign(user.id, role.id); - clock.uninstall(); - await setTimeout(100); + timeService.now += 100; const assignments = await roleAssignmentsRepository.find({ where: { diff --git a/packages/backend/test/unit/S3Service.ts b/packages/backend/test/unit/S3Service.ts index 151f3b826a..4445b0f5f1 100644 --- a/packages/backend/test/unit/S3Service.ts +++ b/packages/backend/test/unit/S3Service.ts @@ -14,28 +14,38 @@ import { UploadPartCommand, } from '@aws-sdk/client-s3'; import { mockClient } from 'aws-sdk-client-mock'; +import { MockInternalEventService } from '../misc/MockInternalEventService.js'; +import type { TestingModule } from '@nestjs/testing'; import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; import { S3Service } from '@/core/S3Service.js'; import { MiMeta } from '@/models/_.js'; -import type { TestingModule } from '@nestjs/testing'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { InternalEventService } from '@/global/InternalEventService.js'; +import { DI } from '@/di-symbols.js'; describe('S3Service', () => { let app: TestingModule; let s3Service: S3Service; + let fakeMeta: MiMeta; const s3Mock = mockClient(S3Client); beforeAll(async () => { app = await Test.createTestingModule({ imports: [GlobalModule, CoreModule], - providers: [S3Service], - }).compile(); + }) + .overrideProvider(InternalEventService).useClass(MockInternalEventService) + .compile(); + + await app.init(); app.enableShutdownHooks(); - s3Service = app.get(S3Service); }); beforeEach(async () => { s3Mock.reset(); + + fakeMeta = Object.create(app.get(DI.meta)); + s3Service = new S3Service(fakeMeta, app.get(HttpRequestService), app.get(InternalEventService)); }); afterAll(async () => { @@ -45,8 +55,9 @@ describe('S3Service', () => { describe('upload', () => { test('upload a file', async () => { s3Mock.on(PutObjectCommand).resolves({}); + fakeMeta.objectStorageRegion = 'us-east-1'; - await s3Service.upload({ objectStorageRegion: 'us-east-1' } as MiMeta, { + await s3Service.upload({ Bucket: 'fake', Key: 'fake', Body: 'x', @@ -58,7 +69,7 @@ describe('S3Service', () => { s3Mock.on(UploadPartCommand).resolves({ ETag: '1' }); s3Mock.on(CompleteMultipartUploadCommand).resolves({ Bucket: 'fake', Key: 'fake' }); - await s3Service.upload({} as MiMeta, { + await s3Service.upload({ Bucket: 'fake', Key: 'fake', Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ @@ -67,22 +78,23 @@ describe('S3Service', () => { test('upload a file error', async () => { s3Mock.on(PutObjectCommand).rejects({ name: 'Fake Error' }); + fakeMeta.objectStorageRegion = 'us-east-1'; - await expect(s3Service.upload({ objectStorageRegion: 'us-east-1' } as MiMeta, { + await expect(s3Service.upload({ Bucket: 'fake', Key: 'fake', Body: 'x', - })).rejects.toThrowError(Error); + })).rejects.toThrow(); }); test('upload a large file error', async () => { s3Mock.on(UploadPartCommand).rejects(); - await expect(s3Service.upload({} as MiMeta, { + await expect(s3Service.upload({ Bucket: 'fake', Key: 'fake', Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ - })).rejects.toThrowError(Error); + })).rejects.toThrow(); }); }); }); diff --git a/packages/backend/test/unit/SigninWithPasskeyApiService.ts b/packages/backend/test/unit/SigninWithPasskeyApiService.ts index 5bc86f6616..2adf57ba52 100644 --- a/packages/backend/test/unit/SigninWithPasskeyApiService.ts +++ b/packages/backend/test/unit/SigninWithPasskeyApiService.ts @@ -9,7 +9,8 @@ import { Test, TestingModule } from '@nestjs/testing'; import { FastifyReply, FastifyRequest } from 'fastify'; import { AuthenticationResponseJSON } from '@simplewebauthn/types'; import { HttpHeader } from 'fastify/types/utils.js'; -import { MockFunctionMetadata, ModuleMocker } from 'jest-mock'; +import { MockMetadata, ModuleMocker } from 'jest-mock'; +import { FakeSkRateLimiterService } from '../misc/FakeSkRateLimiterService.js'; import { MiUser } from '@/models/User.js'; import { MiUserProfile, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; @@ -21,23 +22,11 @@ import { WebAuthnService } from '@/core/WebAuthnService.js'; import { SigninService } from '@/server/api/SigninService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; -import { LimitInfo } from '@/misc/rate-limit-utils.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; +import { ServerModule } from '@/server/ServerModule.js'; const moduleMocker = new ModuleMocker(global); -class FakeLimiter { - public async limit(): Promise { - return { - blocked: false, - remaining: Number.MAX_SAFE_INTEGER, - resetMs: 0, - resetSec: 0, - fullResetMs: 0, - fullResetSec: 0, - }; - } -} - class FakeSigninService { public signin(..._args: any): any { return true; @@ -52,6 +41,7 @@ class DummyFastifyReply { header(_key: HttpHeader, _value: any): void { } } + class DummyFastifyRequest { public ip: string; public body: { credential: any, context: string }; @@ -76,43 +66,46 @@ describe('SigninWithPasskeyApiService', () => { let userProfilesRepository: UserProfilesRepository; let webAuthnService: WebAuthnService; let idService: IdService; + let cacheManagementService: CacheManagementService; let FakeWebauthnVerify: ()=>Promise; async function createUser(data: Partial = {}) { - const user = await usersRepository - .save({ - ...data, - }); - return user; + await usersRepository.insert(data); + return await usersRepository.findOneByOrFail({ id: data.id }); } async function createUserProfile(data: Partial = {}) { - const userProfile = await userProfilesRepository - .save({ ...data }, - ); - return userProfile; + await userProfilesRepository.insert(data); + return await userProfilesRepository.findOneByOrFail({ userId: data.userId }); } beforeAll(async () => { app = await Test.createTestingModule({ - imports: [GlobalModule, CoreModule], - providers: [ - SigninWithPasskeyApiService, - { provide: SkRateLimiterService, useClass: FakeLimiter }, - { provide: SigninService, useClass: FakeSigninService }, - ], + imports: [GlobalModule, CoreModule, ServerModule], }).useMocker((token) => { if (typeof token === 'function') { - const mockMetadata = moduleMocker.getMetadata(token) as MockFunctionMetadata; + const mockMetadata = moduleMocker.getMetadata(token) as MockMetadata; const Mock = moduleMocker.generateFromMetadata(mockMetadata); return new Mock(); } - }).compile(); + }) + .overrideProvider(SkRateLimiterService).useClass(FakeSkRateLimiterService) + .overrideProvider(SigninService).useClass(FakeSigninService) + .compile(); + + await app.init(); + app.enableShutdownHooks(); + passkeyApiService = app.get(SigninWithPasskeyApiService); usersRepository = app.get(DI.usersRepository); userProfilesRepository = app.get(DI.userProfilesRepository); webAuthnService = app.get(WebAuthnService); idService = app.get(IdService); + cacheManagementService = app.get(CacheManagementService); + }); + + afterAll(async () => { + await app.close(); }); beforeEach(async () => { @@ -124,7 +117,7 @@ describe('SigninWithPasskeyApiService', () => { const dummyUser = { id: uid, username: uid, usernameLower: uid.toLowerCase(), uri: null, host: null, - }; + }; const dummyProfile = { userId: uid, password: 'qwerty', @@ -134,8 +127,10 @@ describe('SigninWithPasskeyApiService', () => { await createUserProfile(dummyProfile); }); - afterAll(async () => { - await app.close(); + afterEach(async () => { + await userProfilesRepository.deleteAll(); + await usersRepository.deleteAll(); + cacheManagementService.clear(); }); describe('Get Passkey Options', () => { diff --git a/packages/backend/test/unit/SystemWebhookService.ts b/packages/backend/test/unit/SystemWebhookService.ts index 61187e9f2a..4b253a3d52 100644 --- a/packages/backend/test/unit/SystemWebhookService.ts +++ b/packages/backend/test/unit/SystemWebhookService.ts @@ -19,6 +19,8 @@ import { DI } from '@/di-symbols.js'; import { QueueService } from '@/core/QueueService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { SystemWebhookService } from '@/core/SystemWebhookService.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; +import { CoreModule } from '@/core/CoreModule.js'; describe('SystemWebhookService', () => { let app: TestingModule; @@ -29,6 +31,7 @@ describe('SystemWebhookService', () => { let usersRepository: UsersRepository; let systemWebhooksRepository: SystemWebhooksRepository; let idService: IdService; + let cacheManagementService: CacheManagementService; let queueService: jest.Mocked; // -------------------------------------------------------------------------------------- @@ -61,58 +64,48 @@ describe('SystemWebhookService', () => { // -------------------------------------------------------------------------------------- - async function beforeAllImpl() { + beforeAll(async () => { app = await Test .createTestingModule({ imports: [ GlobalModule, - ], - providers: [ - SystemWebhookService, - IdService, - LoggerService, - GlobalEventService, - { - provide: QueueService, useFactory: () => ({ systemWebhookDeliver: jest.fn() }), - }, - { - provide: ModerationLogService, useFactory: () => ({ log: () => Promise.resolve() }), - }, + CoreModule, ], }) + .overrideProvider(QueueService).useValue({ systemWebhookDeliver: jest.fn() }) + .overrideProvider(ModerationLogService).useValue({ log: () => Promise.resolve() }) .compile(); + await app.init(); + app.enableShutdownHooks(); + usersRepository = app.get(DI.usersRepository); systemWebhooksRepository = app.get(DI.systemWebhooksRepository); service = app.get(SystemWebhookService); idService = app.get(IdService); + cacheManagementService = app.get(CacheManagementService); queueService = app.get(QueueService) as jest.Mocked; + }); - app.enableShutdownHooks(); - } - - async function afterAllImpl() { + afterAll(async () => { await app.close(); - } + }); - async function beforeEachImpl() { + beforeEach(async () => { root = await createUser({ username: 'root', usernameLower: 'root' }); - } + }); - async function afterEachImpl() { - await usersRepository.delete({}); - await systemWebhooksRepository.delete({}); - } + afterEach(async () => { + await usersRepository.deleteAll(); + await systemWebhooksRepository.deleteAll(); + queueService.systemWebhookDeliver.mockReset(); + cacheManagementService.clear(); + }); // -------------------------------------------------------------------------------------- describe('アプリを毎回作り直す必要のないグループ', () => { - beforeAll(beforeAllImpl); - afterAll(afterAllImpl); - beforeEach(beforeEachImpl); - afterEach(afterEachImpl); - describe('fetchSystemWebhooks', () => { test('フィルタなし', async () => { const webhook1 = await createWebhook({ @@ -298,16 +291,6 @@ describe('SystemWebhookService', () => { }); describe('アプリを毎回作り直す必要があるグループ', () => { - beforeEach(async () => { - await beforeAllImpl(); - await beforeEachImpl(); - }); - - afterEach(async () => { - await afterEachImpl(); - await afterAllImpl(); - }); - describe('enqueueSystemWebhook', () => { test('キューに追加成功', async () => { const webhook = await createWebhook({ diff --git a/packages/backend/test/unit/UserSearchService.ts b/packages/backend/test/unit/UserSearchService.ts index a6b331d1cb..7d2054dee0 100644 --- a/packages/backend/test/unit/UserSearchService.ts +++ b/packages/backend/test/unit/UserSearchService.ts @@ -11,8 +11,10 @@ import { FollowingsRepository, InstancesRepository, MiUser, UserProfilesReposito import { IdService } from '@/core/IdService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { DI } from '@/di-symbols.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { genAidx } from '@/misc/id/aidx.js'; +import { CoreModule } from '@/core/CoreModule.js'; describe('UserSearchService', () => { let app: TestingModule; @@ -22,6 +24,7 @@ describe('UserSearchService', () => { let usersRepository: UsersRepository; let followingsRepository: FollowingsRepository; let idService: IdService; + let cacheManagementService: CacheManagementService; let userProfilesRepository: UserProfilesRepository; let root: MiUser; @@ -103,20 +106,16 @@ describe('UserSearchService', () => { .createTestingModule({ imports: [ GlobalModule, + CoreModule, ], - providers: [ - UserSearchService, - { - provide: UserEntityService, useFactory: jest.fn(() => ({ - // とりあえずIDが返れば確認が出来るので - packMany: (value: any) => value, - })), - }, - IdService, - ], + }) + .overrideProvider(UserEntityService).useValue({ + // とりあえずIDが返れば確認が出来るので + packMany: (value: any) => value, }) .compile(); + app.enableShutdownHooks(); await app.init(); instancesRepository = app.get(DI.instancesRepository); @@ -126,6 +125,7 @@ describe('UserSearchService', () => { service = app.get(UserSearchService); idService = app.get(IdService); + cacheManagementService = app.get(CacheManagementService); }); beforeEach(async () => { @@ -143,7 +143,8 @@ describe('UserSearchService', () => { }); afterEach(async () => { - await usersRepository.delete({}); + await usersRepository.deleteAll(); + cacheManagementService.clear(); }); afterAll(async () => { diff --git a/packages/backend/test/unit/UserWebhookService.ts b/packages/backend/test/unit/UserWebhookService.ts index a2a85e9489..3622fa425a 100644 --- a/packages/backend/test/unit/UserWebhookService.ts +++ b/packages/backend/test/unit/UserWebhookService.ts @@ -15,6 +15,8 @@ import { DI } from '@/di-symbols.js'; import { QueueService } from '@/core/QueueService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { UserWebhookService } from '@/core/UserWebhookService.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; +import { CoreModule } from '@/core/CoreModule.js'; describe('UserWebhookService', () => { let app: TestingModule; @@ -25,6 +27,7 @@ describe('UserWebhookService', () => { let usersRepository: UsersRepository; let userWebhooksRepository: WebhooksRepository; let idService: IdService; + let cacheManagementService: CacheManagementService; let queueService: jest.Mocked; // -------------------------------------------------------------------------------------- @@ -58,55 +61,47 @@ describe('UserWebhookService', () => { // -------------------------------------------------------------------------------------- - async function beforeAllImpl() { + beforeAll(async () => { app = await Test .createTestingModule({ imports: [ GlobalModule, - ], - providers: [ - UserWebhookService, - IdService, - LoggerService, - GlobalEventService, - { - provide: QueueService, useFactory: () => ({ userWebhookDeliver: jest.fn() }), - }, + CoreModule, ], }) + .overrideProvider(QueueService).useValue({ userWebhookDeliver: jest.fn() }) .compile(); + await app.init(); + app.enableShutdownHooks(); + usersRepository = app.get(DI.usersRepository); userWebhooksRepository = app.get(DI.webhooksRepository); service = app.get(UserWebhookService); idService = app.get(IdService); + cacheManagementService = app.get(CacheManagementService); queueService = app.get(QueueService) as jest.Mocked; + }); - app.enableShutdownHooks(); - } - - async function afterAllImpl() { + afterAll(async () => { await app.close(); - } + }); - async function beforeEachImpl() { + beforeEach(async () => { root = await createUser({ username: 'root', usernameLower: 'root' }); - } + }); - async function afterEachImpl() { - await usersRepository.delete({}); - await userWebhooksRepository.delete({}); - } + afterEach(async () => { + await usersRepository.deleteAll(); + await userWebhooksRepository.deleteAll(); + queueService.userWebhookDeliver.mockReset(); + cacheManagementService.clear(); + }); // -------------------------------------------------------------------------------------- describe('アプリを毎回作り直す必要のないグループ', () => { - beforeAll(beforeAllImpl); - afterAll(afterAllImpl); - beforeEach(beforeEachImpl); - afterEach(afterEachImpl); - describe('fetchSystemWebhooks', () => { test('フィルタなし', async () => { const webhook1 = await createWebhook({ @@ -243,16 +238,6 @@ describe('UserWebhookService', () => { }); describe('アプリを毎回作り直す必要があるグループ', () => { - beforeEach(async () => { - await beforeAllImpl(); - await beforeEachImpl(); - }); - - afterEach(async () => { - await afterEachImpl(); - await afterAllImpl(); - }); - describe('enqueueUserWebhook', () => { test('キューに追加成功', async () => { const webhook = await createWebhook({ diff --git a/packages/backend/test/unit/UtilityService.ts b/packages/backend/test/unit/UtilityService.ts index f4e92b85a7..d2e405ee77 100644 --- a/packages/backend/test/unit/UtilityService.ts +++ b/packages/backend/test/unit/UtilityService.ts @@ -1,30 +1,38 @@ -import * as assert from 'assert'; -import { Test } from '@nestjs/testing'; -import { jest } from '@jest/globals'; +/* + * SPDX-FileCopyrightText: dakkar and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ -import { CoreModule } from '@/core/CoreModule.js'; -import { UtilityService } from '@/core/UtilityService.js'; -import { MetaService } from '@/core/MetaService.js'; -import { GlobalModule } from '@/GlobalModule.js'; -import { MiMeta } from '@/models/_.js'; -import { DI } from '@/di-symbols.js'; +import * as assert from 'assert'; +import type { MiMeta } from '@/models/_.js'; +import type { Config } from '@/config.js'; import type { SoftwareSuspension } from '@/models/Meta.js'; import type { MiInstance } from '@/models/Instance.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { EnvService } from '@/global/EnvService.js'; describe('UtilityService', () => { let utilityService: UtilityService; - let meta: jest.Mocked; + let meta: MiMeta; - beforeAll(async () => { - const app = await Test.createTestingModule({ - imports: [GlobalModule, CoreModule], - providers: [MetaService], - }) - .overrideProvider(MetaService).useValue({ fetch: jest.fn() }) - .compile(); + beforeEach(() => { + const config = { + url: 'https://example.com', + host: 'example.com', + } as unknown as Config; - utilityService = app.get(UtilityService); - meta = app.get(DI.meta) as jest.Mocked; + meta = { + blockedHosts: [], + silencedHosts: [], + mediaSilencedHosts: [], + federationHosts: [], + bubbleInstances: [], + deliverSuspendedSoftware: [], + federation: 'all', + } as unknown as MiMeta; + + const envService = new EnvService(); + utilityService = new UtilityService(config, meta, envService); }); describe('punyHost', () => { @@ -149,7 +157,7 @@ describe('UtilityService', () => { checkThis( [{ software: 'Test', versionRange: '1.2.3' }], { softwareName: 'Test', softwareVersion: '1-2-3' }, - false, "semver can't parse softwareVersion", + false, 'semver can\'t parse softwareVersion', ); checkThis( [{ software: 'Test', versionRange: '*' }], diff --git a/packages/backend/test/unit/WebhookTestService.ts b/packages/backend/test/unit/WebhookTestService.ts index 736aac40b4..b06be72c9b 100644 --- a/packages/backend/test/unit/WebhookTestService.ts +++ b/packages/backend/test/unit/WebhookTestService.ts @@ -15,6 +15,8 @@ import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { QueueService } from '@/core/QueueService.js'; import { CustomEmojiService } from '@/core/CustomEmojiService.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; +import { CoreModule } from '@/core/CoreModule.js'; describe('WebhookTestService', () => { let app: TestingModule; @@ -28,6 +30,7 @@ describe('WebhookTestService', () => { let userWebhookService: jest.Mocked; let systemWebhookService: jest.Mocked; let idService: IdService; + let cacheManagementService: CacheManagementService; let root: MiUser; let alice: MiUser; @@ -53,44 +56,40 @@ describe('WebhookTestService', () => { app = await Test.createTestingModule({ imports: [ GlobalModule, + CoreModule, ], - providers: [ - WebhookTestService, - IdService, - { - provide: CustomEmojiService, useFactory: () => ({ - populateEmojis: jest.fn(), - }), - }, - { - provide: QueueService, useFactory: () => ({ - systemWebhookDeliver: jest.fn(), - userWebhookDeliver: jest.fn(), - }), - }, - { - provide: UserWebhookService, useFactory: () => ({ - fetchWebhooks: jest.fn(), - }), - }, - { - provide: SystemWebhookService, useFactory: () => ({ - fetchSystemWebhooks: jest.fn(), - }), - }, - ], - }).compile(); + }) + .overrideProvider(CustomEmojiService).useValue({ + populateEmojis: jest.fn(), + }) + .overrideProvider(QueueService).useValue({ + systemWebhookDeliver: jest.fn(), + userWebhookDeliver: jest.fn(), + }) + .overrideProvider(UserWebhookService).useValue({ + fetchWebhooks: jest.fn(), + }) + .overrideProvider(SystemWebhookService).useValue({ + fetchSystemWebhooks: jest.fn(), + }) + .compile(); + + await app.init(); + app.enableShutdownHooks(); usersRepository = app.get(DI.usersRepository); userProfilesRepository = app.get(DI.userProfilesRepository); service = app.get(WebhookTestService); idService = app.get(IdService); + cacheManagementService = app.get(CacheManagementService); queueService = app.get(QueueService) as jest.Mocked; userWebhookService = app.get(UserWebhookService) as jest.Mocked; systemWebhookService = app.get(SystemWebhookService) as jest.Mocked; + }); - app.enableShutdownHooks(); + afterAll(async () => { + await app.close(); }); beforeEach(async () => { @@ -111,12 +110,9 @@ describe('WebhookTestService', () => { userWebhookService.fetchWebhooks.mockClear(); systemWebhookService.fetchSystemWebhooks.mockClear(); - await usersRepository.delete({}); - await userProfilesRepository.delete({}); - }); - - afterAll(async () => { - await app.close(); + await userProfilesRepository.deleteAll(); + await usersRepository.deleteAll(); + cacheManagementService.clear(); }); // -------------------------------------------------------------------------------------- diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index ff93e1be07..eccad42633 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -2,19 +2,17 @@ * SPDX-FileCopyrightText: syuilo and misskey-project * SPDX-License-Identifier: AGPL-3.0-only */ + process.env.NODE_ENV = 'test'; import * as assert from 'assert'; import { generateKeyPair } from 'crypto'; -import { Test } from '@nestjs/testing'; +import { Test, TestingModule } from '@nestjs/testing'; import { jest } from '@jest/globals'; - -import { NoOpCacheService } from '../misc/noOpCaches.js'; -import { FakeInternalEventService } from '../misc/FakeInternalEventService.js'; +import { MockApResolverService } from '../misc/MockApResolverService.js'; +import { MockConsole } from '../misc/MockConsole.js'; import type { Config } from '@/config.js'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; -import { CacheService } from '@/core/CacheService.js'; import { ApImageService } from '@/core/activitypub/models/ApImageService.js'; import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; @@ -25,16 +23,16 @@ import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { LoggerService } from '@/core/LoggerService.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; +import { ApResolverService } from '@/core/activitypub/ApResolverService.js'; import type { IActor, IApDocument, ICollection, IObject, IPost } from '@/core/activitypub/type.js'; -import { MiMeta, MiNote, MiUser, MiUserKeypair, UserProfilesRepository, UserPublickeysRepository } from '@/models/_.js'; +import { MiMeta, MiNote, MiUser, MiUserKeypair, UserProfilesRepository, UserPublickeysRepository, UserKeypairsRepository, UsersRepository, NotesRepository } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; import { DownloadService } from '@/core/DownloadService.js'; import { genAidx } from '@/misc/id/aidx.js'; import { IdService } from '@/core/IdService.js'; import { MockResolver } from '../misc/mock-resolver.js'; -import { UserKeypairService } from '@/core/UserKeypairService.js'; -import { MemoryKVCache } from '@/misc/cache.js'; const host = 'https://host1.test'; @@ -94,6 +92,7 @@ async function createRandomRemoteUser( } describe('ActivityPub', () => { + let app: TestingModule; let userProfilesRepository: UserProfilesRepository; let imageService: ApImageService; let noteService: ApNoteService; @@ -103,8 +102,12 @@ describe('ActivityPub', () => { let resolver: MockResolver; let idService: IdService; let userPublickeysRepository: UserPublickeysRepository; - let userKeypairService: UserKeypairService; + let userKeypairsRepository: UserKeypairsRepository; + let usersRepository: UsersRepository; let config: Config; + let cacheManagementService: CacheManagementService; + let mockConsole: MockConsole; + let notesRepository: NotesRepository; const metaInitial = { id: 'x', @@ -147,7 +150,7 @@ describe('ActivityPub', () => { } beforeAll(async () => { - const app = await Test.createTestingModule({ + app = await Test.createTestingModule({ imports: [GlobalModule, CoreModule], }) .overrideProvider(DownloadService).useValue({ @@ -157,9 +160,9 @@ describe('ActivityPub', () => { }; }, }) - .overrideProvider(DI.meta).useFactory({ factory: () => meta }) - .overrideProvider(CacheService).useClass(NoOpCacheService) - .overrideProvider(InternalEventService).useClass(FakeInternalEventService) + .overrideProvider(DI.meta).useValue(meta) + .overrideProvider(ApResolverService).useClass(MockApResolverService) + .overrideProvider(DI.console).useClass(MockConsole) .compile(); await app.init(); @@ -172,18 +175,30 @@ describe('ActivityPub', () => { rendererService = app.get(ApRendererService); imageService = app.get(ApImageService); jsonLdService = app.get(JsonLdService); - resolver = new MockResolver(await app.resolve(LoggerService)); + resolver = app.get(ApResolverService).resolver; idService = app.get(IdService); userPublickeysRepository = app.get(DI.userPublickeysRepository); - userKeypairService = app.get(UserKeypairService); + userKeypairsRepository = app.get(DI.userKeypairsRepository); + usersRepository = app.get(DI.usersRepository); config = app.get(DI.config); - - // Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error - const federatedInstanceService = app.get(FederatedInstanceService); - jest.spyOn(federatedInstanceService, 'fetch').mockImplementation(() => new Promise(() => { })); + cacheManagementService = app.get(CacheManagementService); + mockConsole = app.get(DI.console); + notesRepository = app.get(DI.notesRepository); }); - beforeEach(() => { + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + // This will cascade-delete everything else + await usersRepository.deleteAll(); + + // Clear all caches app-wide + cacheManagementService.clear(); + + // Reset mocks + mockConsole.mockReset(); resolver.clear(); }); @@ -204,6 +219,7 @@ describe('ActivityPub', () => { const user = await personService.createPerson(actor.id, resolver); + mockConsole.assertNoErrors(); assert.deepStrictEqual(user.uri, actor.id); assert.deepStrictEqual(user.username, actor.preferredUsername); assert.deepStrictEqual(user.inbox, actor.inbox); @@ -215,6 +231,7 @@ describe('ActivityPub', () => { const note = await noteService.createNote(post.id, undefined, resolver, true); + mockConsole.assertNoErrors(); assert.deepStrictEqual(note?.uri, post.id); assert.deepStrictEqual(note.visibility, 'public'); assert.deepStrictEqual(note.text, post.content); @@ -250,30 +267,45 @@ describe('ActivityPub', () => { }); describe('Collection visibility', () => { - test('Public following/followers', async () => { - const actor = createRandomActor(); - actor.following = { - id: `${actor.id}/following`, - type: 'OrderedCollection', - totalItems: 0, - first: `${actor.id}/following?page=1`, + function createPublicTest(inline: boolean) { + return async () => { + const actor = createRandomActor(); + const following = { + id: `${actor.id}/following`, + type: 'OrderedCollection', + totalItems: 0, + first: `${actor.id}/following?page=1`, + } as const; + const followers = { + id: `${actor.id}/followers`, + type: 'OrderedCollection', + totalItems: 0, + first: `${actor.id}/followers?page=1`, + } as const; + + if (inline) { + actor.following = following; + actor.followers = followers; + } else { + actor.following = following.id; + actor.followers = followers.id; + } + + resolver.register(actor.id, actor); + resolver.register(following.id, following); + resolver.register(followers.id, followers); + + const user = await personService.createPerson(actor.id, resolver); + const userProfile = await userProfilesRepository.findOneByOrFail({ userId: user.id }); + + mockConsole.assertNoErrors(); + assert.deepStrictEqual(userProfile.followingVisibility, 'public'); + assert.deepStrictEqual(userProfile.followersVisibility, 'public'); }; - actor.followers = `${actor.id}/followers`; + } - resolver.register(actor.id, actor); - resolver.register(actor.followers, { - id: actor.followers, - type: 'OrderedCollection', - totalItems: 0, - first: `${actor.followers}?page=1`, - }); - - const user = await personService.createPerson(actor.id, resolver); - const userProfile = await userProfilesRepository.findOneByOrFail({ userId: user.id }); - - assert.deepStrictEqual(userProfile.followingVisibility, 'public'); - assert.deepStrictEqual(userProfile.followersVisibility, 'public'); - }); + test('Public following/followers (URI)', createPublicTest(false)); + test('Public following/followers (inline)', createPublicTest(true)); test('Private following/followers', async () => { const actor = createRandomActor(); @@ -327,6 +359,8 @@ describe('ActivityPub', () => { assert.strictEqual(note.text, 'test test foo'); assert.strictEqual(note.uri, item.id); } + + mockConsole.assertNoErrors(); }); test('Fetch featured notes from IActor pointing to another remote server', async () => { @@ -528,15 +562,15 @@ describe('ActivityPub', () => { name: 'Test Author', isCat: true, requireSigninToViewContents: true, - makeNotesFollowersOnlyBefore: new Date(2025, 2, 20).valueOf(), - makeNotesHiddenBefore: new Date(2025, 2, 21).valueOf(), + makeNotesFollowersOnlyBefore: new Date(2025, 2, 20).valueOf() / 1000, + makeNotesHiddenBefore: new Date(2025, 2, 21).valueOf() / 1000, isLocked: true, isExplorable: true, hideOnlineStatus: true, noindex: true, enableRss: true, - }) as MiLocalUser; + await usersRepository.insert(author); const [publicKey, privateKey] = await new Promise<[string, string]>((res, rej) => generateKeyPair('rsa', { @@ -560,7 +594,7 @@ describe('ActivityPub', () => { publicKey, privateKey, }); - (userKeypairService as unknown as { cache: MemoryKVCache }).cache.set(author.id, keypair); + await userKeypairsRepository.insert(keypair); note = new MiNote({ id: idService.gen(), @@ -585,6 +619,7 @@ describe('ActivityPub', () => { tags: [], hasPoll: false, }); + await notesRepository.insert(note); }); describe('renderNote', () => { @@ -823,6 +858,7 @@ describe('ActivityPub', () => { expect(publicKey).not.toBeNull(); expect(publicKey?.keyPem).toBe('key material'); + mockConsole.assertNoErrors(); }); it('should accept SocialHome actor', async () => { @@ -871,6 +907,7 @@ describe('ActivityPub', () => { expect(user.uri).toBe(actor.id); expect(publicKey).not.toBeNull(); + mockConsole.assertNoErrors(); }); }); }); diff --git a/packages/backend/test/unit/ap-request.ts b/packages/backend/test/unit/ap-request.ts index 11364e1735..894caa1528 100644 --- a/packages/backend/test/unit/ap-request.ts +++ b/packages/backend/test/unit/ap-request.ts @@ -40,7 +40,7 @@ describe('ap-request', () => { 'User-Agent': 'UA', }; - const req = ApRequestCreator.createSignedPost({ key, url, body, additionalHeaders: headers }); + const req = ApRequestCreator.createSignedPost({ key, url, body, additionalHeaders: headers, now: Date.now() }); const parsed = buildParsedSignature(req.signingString, req.signature, 'rsa-sha256'); @@ -56,7 +56,7 @@ describe('ap-request', () => { 'User-Agent': 'UA', }; - const req = ApRequestCreator.createSignedGet({ key, url, additionalHeaders: headers }); + const req = ApRequestCreator.createSignedGet({ key, url, additionalHeaders: headers, now: Date.now() }); const parsed = buildParsedSignature(req.signingString, req.signature, 'rsa-sha256'); diff --git a/packages/backend/test/unit/chart.ts b/packages/backend/test/unit/chart.ts index 9dedd3a79d..17307a6ccc 100644 --- a/packages/backend/test/unit/chart.ts +++ b/packages/backend/test/unit/chart.ts @@ -3,12 +3,16 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { MockConsole } from '../misc/MockConsole.js'; + process.env.NODE_ENV = 'test'; import * as assert from 'assert'; -import { jest } from '@jest/globals'; -import * as lolex from '@sinonjs/fake-timers'; import { DataSource } from 'typeorm'; +import { Test, TestingModule } from '@nestjs/testing'; +import { GodOfTimeService } from '../misc/GodOfTimeService.js'; +import { MockRedis } from '../misc/MockRedis.js'; +import { GlobalModule } from '@/GlobalModule.js'; import TestChart from '@/core/chart/charts/test.js'; import TestGroupedChart from '@/core/chart/charts/test-grouped.js'; import TestUniqueChart from '@/core/chart/charts/test-unique.js'; @@ -17,70 +21,72 @@ import { entity as TestChartEntity } from '@/core/chart/charts/entities/test.js' import { entity as TestGroupedChartEntity } from '@/core/chart/charts/entities/test-grouped.js'; import { entity as TestUniqueChartEntity } from '@/core/chart/charts/entities/test-unique.js'; import { entity as TestIntersectionChartEntity } from '@/core/chart/charts/entities/test-intersection.js'; -import { loadConfig } from '@/config.js'; -import type { AppLockService } from '@/core/AppLockService.js'; +import { AppLockService } from '@/core/AppLockService.js'; import Logger from '@/logger.js'; +import { CoreModule } from '@/core/CoreModule.js'; +import { DI } from '@/di-symbols.js'; +import { TimeService } from '@/global/TimeService.js'; +import { LoggerService } from '@/core/LoggerService.js'; describe('Chart', () => { - const config = loadConfig(); - const appLockService = { - getChartInsertLock: () => () => Promise.resolve(() => {}), - } as unknown as jest.Mocked; - - let db: DataSource | undefined; + let app: TestingModule; + let db: DataSource; + let appLockService: AppLockService; + let logger: Logger; + let redis: MockRedis; let testChart: TestChart; let testGroupedChart: TestGroupedChart; let testUniqueChart: TestUniqueChart; let testIntersectionChart: TestIntersectionChart; - let clock: lolex.InstalledClock; + let clock: GodOfTimeService; - beforeEach(async () => { - if (db) db.destroy(); + beforeAll(async () => { + app = await Test.createTestingModule({ + imports: [GlobalModule, CoreModule], + }) + .overrideProvider(DI.redis).useClass(MockRedis) + .overrideProvider(TimeService).useClass(GodOfTimeService) + .overrideProvider(DI.console).useClass(MockConsole) + .compile(); - db = new DataSource({ - type: 'postgres', - host: config.db.host, - port: config.db.port, - username: config.db.user, - password: config.db.pass, - database: config.db.db, - extra: { - statement_timeout: 1000 * 10, - ...config.db.extra, - }, - synchronize: true, - dropSchema: true, - maxQueryExecutionTime: 300, - entities: [ - TestChartEntity.hour, TestChartEntity.day, - TestGroupedChartEntity.hour, TestGroupedChartEntity.day, - TestUniqueChartEntity.hour, TestUniqueChartEntity.day, - TestIntersectionChartEntity.hour, TestIntersectionChartEntity.day, - ], - migrations: ['../../migration/*.js'], - }); + logger = app.get(LoggerService).getLogger('chart'); + appLockService = app.get(AppLockService); + redis = app.get(DI.redis); + db = app.get(DI.db); - await db.initialize(); + clock = app.get(TimeService); + clock.resetTo(Date.UTC(2000, 0, 1, 0, 0, 0)); - const logger = new Logger('chart'); // TODO: モックにする - testChart = new TestChart(db, appLockService, logger); - testGroupedChart = new TestGroupedChart(db, appLockService, logger); - testUniqueChart = new TestUniqueChart(db, appLockService, logger); - testIntersectionChart = new TestIntersectionChart(db, appLockService, logger); - - clock = lolex.install({ - now: new Date(Date.UTC(2000, 0, 1, 0, 0, 0)), - shouldClearNativeTimers: true, - }); - }); - - afterEach(() => { - clock.uninstall(); + await app.init(); + app.enableShutdownHooks(); }); afterAll(async () => { - if (db) await db.destroy(); + await app.close(); + }); + + beforeEach(async () => { + clock.resetTo(Date.UTC(2000, 0, 1, 0, 0, 0)); + redis.mockReset(); + + testChart = new TestChart(db, appLockService, clock, logger); + testGroupedChart = new TestGroupedChart(db, appLockService, clock, logger); + testUniqueChart = new TestUniqueChart(db, appLockService, clock, logger); + testIntersectionChart = new TestIntersectionChart(db, appLockService, clock, logger); + }); + + afterEach(async () => { + const entities = [ + TestChartEntity.hour, TestChartEntity.day, + TestGroupedChartEntity.hour, TestGroupedChartEntity.day, + TestUniqueChartEntity.hour, TestUniqueChartEntity.day, + TestIntersectionChartEntity.hour, TestIntersectionChartEntity.day, + ]; + + for (const entity of entities) { + await db.getRepository(entity).deleteAll(); + } }); test('Can updates', async () => { @@ -208,7 +214,7 @@ describe('Chart', () => { await testChart.increment(); await testChart.save(); - clock.tick('01:00:00'); + clock.tick({ hours: 1 }); await testChart.increment(); await testChart.save(); @@ -268,7 +274,7 @@ describe('Chart', () => { await testChart.increment(); await testChart.save(); - clock.tick('02:00:00'); + clock.tick({ hours: 2 }); await testChart.increment(); await testChart.save(); @@ -298,7 +304,7 @@ describe('Chart', () => { await testChart.increment(); await testChart.save(); - clock.tick('05:00:00'); + clock.tick({ hours: 5 }); const chartHours = await testChart.getChart('hour', 3, null); const chartDays = await testChart.getChart('day', 3, null); @@ -326,7 +332,7 @@ describe('Chart', () => { await testChart.increment(); await testChart.save(); - clock.tick('05:00:00'); + clock.tick({ hours: 5 }); await testChart.increment(); await testChart.save(); @@ -355,7 +361,7 @@ describe('Chart', () => { await testChart.increment(); await testChart.save(); - clock.tick('01:00:00'); + clock.tick({ hours: 1 }); await testChart.increment(); await testChart.save(); @@ -381,12 +387,12 @@ describe('Chart', () => { }); test('Can specify offset (floor time)', async () => { - clock.tick('00:30:00'); + clock.tick({ minutes: 30 }); await testChart.increment(); await testChart.save(); - clock.tick('01:30:00'); + clock.tick({ hours: 1, minutes: 30 }); await testChart.increment(); await testChart.save(); @@ -552,7 +558,7 @@ describe('Chart', () => { await testChart.increment(); await testChart.save(); - clock.tick('01:00:00'); + clock.tick({ hours: 1 }); testChart.total = 100; diff --git a/packages/backend/test/unit/core/HttpRequestService.ts b/packages/backend/test/unit/core/HttpRequestService.ts index ccce32ffee..fecb4ca14f 100644 --- a/packages/backend/test/unit/core/HttpRequestService.ts +++ b/packages/backend/test/unit/core/HttpRequestService.ts @@ -4,21 +4,24 @@ */ import { describe, jest } from '@jest/globals'; +import { MockConsole } from '../../misc/MockConsole.js'; import type { Mock } from 'jest-mock'; import type { PrivateNetwork } from '@/config.js'; import type { Socket } from 'net'; import { HttpRequestService, isAllowedPrivateIp, isPrivateUrl, resolveIp, validateSocketConnect } from '@/core/HttpRequestService.js'; import { parsePrivateNetworks } from '@/config.js'; +import Logger from '@/logger.js'; describe(HttpRequestService, () => { let allowedPrivateNetworks: PrivateNetwork[] | undefined; beforeEach(() => { + const logger = new Logger('mock', undefined, undefined, undefined, new MockConsole()); allowedPrivateNetworks = parsePrivateNetworks([ '10.0.0.1/32', { network: '127.0.0.1/32', ports: [1] }, { network: '127.0.0.1/32', ports: [3, 4, 5] }, - ]); + ], logger); }); describe(isAllowedPrivateIp, () => { diff --git a/packages/backend/test/unit/core/activitypub/ApUtilityService.ts b/packages/backend/test/unit/core/activitypub/ApUtilityService.ts index 7b564b1fdd..b0f9666bb0 100644 --- a/packages/backend/test/unit/core/activitypub/ApUtilityService.ts +++ b/packages/backend/test/unit/core/activitypub/ApUtilityService.ts @@ -4,7 +4,7 @@ */ import type { IObject } from '@/core/activitypub/type.js'; -import type { EnvService } from '@/core/EnvService.js'; +import type { EnvService } from '@/global/EnvService.js'; import type { MiMeta } from '@/models/Meta.js'; import type { Config } from '@/config.js'; import type { LoggerService } from '@/core/LoggerService.js'; diff --git a/packages/backend/test/unit/entities/UserEntityService.ts b/packages/backend/test/unit/entities/UserEntityService.ts index 3eda3eee01..6566d6e8b3 100644 --- a/packages/backend/test/unit/entities/UserEntityService.ts +++ b/packages/backend/test/unit/entities/UserEntityService.ts @@ -4,9 +4,8 @@ */ import { Test, TestingModule } from '@nestjs/testing'; -import { FakeInternalEventService } from '../../misc/FakeInternalEventService.js'; -import { NoOpCacheService } from '../../misc/noOpCaches.js'; import type { MiUser } from '@/models/User.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; @@ -21,39 +20,6 @@ import { UsersRepository, } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; -import { AvatarDecorationService } from '@/core/AvatarDecorationService.js'; -import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; -import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; -import { PageEntityService } from '@/core/entities/PageEntityService.js'; -import { CustomEmojiService } from '@/core/CustomEmojiService.js'; -import { AnnouncementService } from '@/core/AnnouncementService.js'; -import { RoleService } from '@/core/RoleService.js'; -import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; -import { IdService } from '@/core/IdService.js'; -import { UtilityService } from '@/core/UtilityService.js'; -import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; -import { ModerationLogService } from '@/core/ModerationLogService.js'; -import { GlobalEventService } from '@/core/GlobalEventService.js'; -import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; -import { MetaService } from '@/core/MetaService.js'; -import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; -import { CacheService } from '@/core/CacheService.js'; -import { ApResolverService } from '@/core/activitypub/ApResolverService.js'; -import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; -import { ApImageService } from '@/core/activitypub/models/ApImageService.js'; -import { ApMfmService } from '@/core/activitypub/ApMfmService.js'; -import { MfmService } from '@/core/MfmService.js'; -import { HashtagService } from '@/core/HashtagService.js'; -import UsersChart from '@/core/chart/charts/users.js'; -import { ChartLoggerService } from '@/core/chart/ChartLoggerService.js'; -import InstanceChart from '@/core/chart/charts/instance.js'; -import { ApLoggerService } from '@/core/activitypub/ApLoggerService.js'; -import { AccountMoveService } from '@/core/AccountMoveService.js'; -import { ReactionService } from '@/core/ReactionService.js'; -import { NotificationService } from '@/core/NotificationService.js'; -import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; -import { ChatService } from '@/core/ChatService.js'; -import { InternalEventService } from '@/core/InternalEventService.js'; process.env.NODE_ENV = 'test'; @@ -69,6 +35,7 @@ describe('UserEntityService', () => { let blockingRepository: BlockingsRepository; let mutingRepository: MutingsRepository; let renoteMutingsRepository: RenoteMutingsRepository; + let cacheManagementService: CacheManagementService; async function createUser(userData: Partial = {}, profileData: Partial = {}) { const un = secureRndstr(16); @@ -86,6 +53,7 @@ describe('UserEntityService', () => { userId: user.id, }); + cacheManagementService.clear(); return user; } @@ -96,6 +64,7 @@ describe('UserEntityService', () => { targetUserId: target.id, memo, }); + cacheManagementService.clear(); } async function follow(follower: MiUser, followee: MiUser) { @@ -104,6 +73,7 @@ describe('UserEntityService', () => { followerId: follower.id, followeeId: followee.id, }); + cacheManagementService.clear(); } async function requestFollow(requester: MiUser, requestee: MiUser) { @@ -112,6 +82,7 @@ describe('UserEntityService', () => { followerId: requester.id, followeeId: requestee.id, }); + cacheManagementService.clear(); } async function block(blocker: MiUser, blockee: MiUser) { @@ -120,6 +91,7 @@ describe('UserEntityService', () => { blockerId: blocker.id, blockeeId: blockee.id, }); + cacheManagementService.clear(); } async function mute(mutant: MiUser, mutee: MiUser) { @@ -128,6 +100,7 @@ describe('UserEntityService', () => { muterId: mutant.id, muteeId: mutee.id, }); + cacheManagementService.clear(); } async function muteRenote(mutant: MiUser, mutee: MiUser) { @@ -136,6 +109,7 @@ describe('UserEntityService', () => { muterId: mutant.id, muteeId: mutee.id, }); + cacheManagementService.clear(); } function randomIntRange(weight = 10) { @@ -143,53 +117,11 @@ describe('UserEntityService', () => { } beforeAll(async () => { - const services = [ - UserEntityService, - ApPersonService, - NoteEntityService, - PageEntityService, - CustomEmojiService, - AnnouncementService, - RoleService, - FederatedInstanceService, - IdService, - AvatarDecorationService, - UtilityService, - EmojiEntityService, - ModerationLogService, - GlobalEventService, - DriveFileEntityService, - MetaService, - FetchInstanceMetadataService, - CacheService, - ApResolverService, - ApNoteService, - ApImageService, - ApMfmService, - MfmService, - HashtagService, - UsersChart, - ChartLoggerService, - InstanceChart, - ApLoggerService, - AccountMoveService, - ReactionService, - ReactionsBufferingService, - NotificationService, - ChatService, - InternalEventService, - ]; - app = await Test.createTestingModule({ imports: [GlobalModule, CoreModule], - providers: [ - ...services, - ...services.map(x => ({ provide: x.name, useExisting: x })), - ], }) - .overrideProvider(InternalEventService).useClass(FakeInternalEventService) - .overrideProvider(CacheService).useClass(NoOpCacheService) .compile(); + await app.init(); app.enableShutdownHooks(); @@ -202,12 +134,25 @@ describe('UserEntityService', () => { blockingRepository = app.get(DI.blockingsRepository); mutingRepository = app.get(DI.mutingsRepository); renoteMutingsRepository = app.get(DI.renoteMutingsRepository); + cacheManagementService = app.get(CacheManagementService); }); afterAll(async () => { await app.close(); }); + afterEach(async () => { + await userProfileRepository.deleteAll(); + await userMemosRepository.deleteAll(); + await followingRepository.deleteAll(); + await followingRequestRepository.deleteAll(); + await blockingRepository.deleteAll(); + await mutingRepository.deleteAll(); + await renoteMutingsRepository.deleteAll(); + await usersRepository.deleteAll(); + cacheManagementService.clear(); + }); + test('UserLite', async() => { const me = await createUser(); const who = await createUser(); diff --git a/packages/backend/test/unit/global/CacheManagementServiceTests.ts b/packages/backend/test/unit/global/CacheManagementServiceTests.ts new file mode 100644 index 0000000000..948ef2da3f --- /dev/null +++ b/packages/backend/test/unit/global/CacheManagementServiceTests.ts @@ -0,0 +1,206 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { jest } from '@jest/globals'; +import { MockRedis } from '../../misc/MockRedis.js'; +import { GodOfTimeService } from '../../misc/GodOfTimeService.js'; +import { MockInternalEventService } from '../../misc/MockInternalEventService.js'; +import { CacheManagementService, type Manager, GC_INTERVAL } from '@/global/CacheManagementService.js'; +import { MemoryKVCache } from '@/misc/cache.js'; + +describe(CacheManagementService, () => { + let timeService: GodOfTimeService; + let redisClient: MockRedis; + let internalEventService: MockInternalEventService; + + let serviceUnderTest: CacheManagementService; + let internalsUnderTest: { managedCaches: Set }; + + beforeAll(() => { + timeService = new GodOfTimeService(); + redisClient = new MockRedis(timeService); + internalEventService = new MockInternalEventService( { host: 'example.com' }); + }); + + afterAll(() => { + internalEventService.dispose(); + redisClient.disconnect(); + }); + + beforeEach(() => { + timeService.resetToNow(); + redisClient.mockReset(); + internalEventService.mockReset(); + + serviceUnderTest = new CacheManagementService(redisClient, timeService, internalEventService); + internalsUnderTest = { managedCaches: Reflect.get(serviceUnderTest, 'managedCaches') }; + }); + + afterEach(() => { + serviceUnderTest.dispose(); + }); + + function createCache(): MemoryKVCache { + // Cast to allow access to managed functions, for spying purposes. + return serviceUnderTest.createMemoryKVCache('test', Infinity) as MemoryKVCache; + } + + describe('createMemoryKVCache', () => testCreate('createMemoryKVCache', 'memoryKV', { lifetime: Infinity })); + describe('createMemorySingleCache', () => testCreate('createMemorySingleCache', 'memorySingle', { lifetime: Infinity })); + describe('createRedisKVCache', () => testCreate('createRedisKVCache', 'redisKV', { lifetime: Infinity, memoryCacheLifetime: Infinity })); + describe('createRedisSingleCache', () => testCreate('createRedisSingleCache', 'redisSingle', { lifetime: Infinity, memoryCacheLifetime: Infinity })); + describe('createQuantumKVCache', () => testCreate('createQuantumKVCache', 'quantumKV', { lifetime: Infinity, fetcher: () => { throw new Error('not implement'); } })); + + describe('clear', () => { + testClear('clear', false); + testGC('clear', false, true, false); + }); + describe('dispose', () => { + testClear('dispose', true); + testGC('dispose', false, false, true); + }); + describe('onApplicationShutdown', () => { + testClear('onApplicationShutdown', true); + testGC('onApplicationShutdown', false, false, true); + }); + describe('gc', () => testGC('gc', true, true, false)); + + function testCreate(func: Func, ...args: Parameters) { + // @ts-expect-error TypeScript bug: https://github.com/microsoft/TypeScript/issues/57322 + const act = () => serviceUnderTest[func](...args); + + it('should construct a cache', () => { + const cache = act(); + + expect(cache).not.toBeNull(); + }); + + it('should track reference', () => { + const cache = act(); + + expect(internalsUnderTest.managedCaches.values()).toContain(cache); + }); + + it('should start GC timer', () => { + const cache = act(); + const gc = jest.spyOn(cache as unknown as { gc(): void }, 'gc'); + + timeService.tick({ milliseconds: GC_INTERVAL * 3 }); + + expect(gc).toHaveBeenCalledTimes(3); + }); + + it('should throw if name is duplicate', () => { + act(); + + expect(() => act()).toThrow(); + }); + } + + function testClear(func: 'clear' | 'dispose' | 'onApplicationShutdown', shouldDispose: boolean) { + const act = async () => await serviceUnderTest[func](); + + it('should clear managed caches', async () => { + const cache = createCache(); + const clear = jest.spyOn(cache, 'clear'); + + await act(); + + expect(clear).toHaveBeenCalled(); + }); + + it(`should${shouldDispose ? ' ' : ' not '}dispose managed caches`, async () => { + const cache = createCache(); + const dispose = jest.spyOn(cache, 'dispose'); + + await act(); + + if (shouldDispose) { + expect(dispose).toHaveBeenCalled(); + } else { + expect(dispose).not.toHaveBeenCalled(); + } + }); + + it('should not error with nothing to do', async () => { + await act(); + }); + + it('should be callable multiple times', async () => { + const cache = createCache(); + const clear = jest.spyOn(cache, 'clear'); + + await act(); + await act(); + await act(); + + const expected = shouldDispose ? 1 : 3; + expect(clear).toHaveBeenCalledTimes(expected); + }); + + it(`should${shouldDispose ? ' ' : ' not '}deref caches`, async () => { + const cache = createCache(); + + await act(); + + if (shouldDispose) { + expect(internalsUnderTest.managedCaches.values()).not.toContain(cache); + } else { + expect(internalsUnderTest.managedCaches.values()).toContain(cache); + } + }); + + it(`should${shouldDispose ? ' ' : ' not '}reset cache list`, async () => { + createCache(); + + await act(); + + if (shouldDispose) { + expect(internalsUnderTest.managedCaches.size).toBe(0); + } else { + expect(internalsUnderTest.managedCaches.size).not.toBe(0); + } + }); + } + + function testGC(func: 'clear' | 'dispose' | 'onApplicationShutdown' | 'gc', shouldFire: boolean, shouldReset: boolean, shouldStop: boolean) { + const expectedCalls = + shouldStop + ? shouldFire + ? 1 + : 0 + : shouldFire + ? shouldReset + ? 2 + : 3 + : shouldReset + ? 1 + : 2 + ; + + const testName = 'should ' + [ + shouldFire ? 'trigger' : 'not trigger', + shouldReset ? 'reset' : 'not reset', + shouldStop ? 'and stop' : 'and not stop', + ].join(', ') + ' GC'; + + const arrange = () => jest.spyOn(createCache(), 'gc'); + const act = () => { + timeService.tick({ milliseconds: GC_INTERVAL - 1 }); + serviceUnderTest[func](); + timeService.tick({ milliseconds: 1 }); + timeService.tick({ milliseconds: GC_INTERVAL }); + }; + const assert = (spy: ReturnType) => { + expect(spy).toHaveBeenCalledTimes(expectedCalls); + }; + + it(testName, () => { + const spy = arrange(); + act(); + assert(spy); + }); + } +}); diff --git a/packages/backend/test/unit/misc/QuantumKVCache.ts b/packages/backend/test/unit/misc/QuantumKVCache.ts deleted file mode 100644 index 92792171be..0000000000 --- a/packages/backend/test/unit/misc/QuantumKVCache.ts +++ /dev/null @@ -1,799 +0,0 @@ -/* - * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { jest } from '@jest/globals'; -import { FakeInternalEventService } from '../../misc/FakeInternalEventService.js'; -import { QuantumKVCache, QuantumKVOpts } from '@/misc/QuantumKVCache.js'; - -describe(QuantumKVCache, () => { - let fakeInternalEventService: FakeInternalEventService; - let madeCaches: { dispose: () => void }[]; - - function makeCache(opts?: Partial> & { name?: string }): QuantumKVCache { - const _opts = { - name: 'test', - lifetime: Infinity, - fetcher: () => { throw new Error('not implemented'); }, - } satisfies QuantumKVOpts & { name: string }; - - if (opts) { - Object.assign(_opts, opts); - } - - const cache = new QuantumKVCache(fakeInternalEventService, _opts.name, _opts); - madeCaches.push(cache); - return cache; - } - - beforeEach(() => { - madeCaches = []; - fakeInternalEventService = new FakeInternalEventService(); - }); - - afterEach(() => { - madeCaches.forEach(cache => { - cache.dispose(); - }); - }); - - it('should connect on construct', () => { - makeCache(); - - expect(fakeInternalEventService._calls).toContainEqual(['on', ['quantumCacheUpdated', expect.anything(), { ignoreLocal: true }]]); - }); - - it('should disconnect on dispose', () => { - const cache = makeCache(); - - cache.dispose(); - - const callback = fakeInternalEventService._calls - .find(c => c[0] === 'on' && c[1][0] === 'quantumCacheUpdated') - ?.[1][1]; - expect(fakeInternalEventService._calls).toContainEqual(['off', ['quantumCacheUpdated', callback]]); - }); - - it('should store in memory cache', async () => { - const cache = makeCache(); - - await cache.set('foo', 'bar'); - await cache.set('alpha', 'omega'); - - const result1 = await cache.get('foo'); - const result2 = await cache.get('alpha'); - - expect(result1).toBe('bar'); - expect(result2).toBe('omega'); - }); - - it('should emit event when storing', async () => { - const cache = makeCache({ name: 'fake' }); - - await cache.set('foo', 'bar'); - - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); - }); - - it('should call onChanged when storing', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - onChanged: fakeOnChanged, - }); - - await cache.set('foo', 'bar'); - - expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); - }); - - it('should not emit event when storing unchanged value', async () => { - const cache = makeCache({ name: 'fake' }); - - await cache.set('foo', 'bar'); - await cache.set('foo', 'bar'); - - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); - }); - - it('should not call onChanged when storing unchanged value', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - onChanged: fakeOnChanged, - }); - - await cache.set('foo', 'bar'); - await cache.set('foo', 'bar'); - - expect(fakeOnChanged).toHaveBeenCalledTimes(1); - }); - - it('should fetch an unknown value', async () => { - const cache = makeCache({ - name: 'fake', - fetcher: key => `value#${key}`, - }); - - const result = await cache.fetch('foo'); - - expect(result).toBe('value#foo'); - }); - - it('should store fetched value in memory cache', async () => { - const cache = makeCache({ - name: 'fake', - fetcher: key => `value#${key}`, - }); - - await cache.fetch('foo'); - - const result = cache.has('foo'); - expect(result).toBe(true); - }); - - it('should call onChanged when fetching', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - fetcher: key => `value#${key}`, - onChanged: fakeOnChanged, - }); - - await cache.fetch('foo'); - - expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); - }); - - it('should not emit event when fetching', async () => { - const cache = makeCache({ - name: 'fake', - fetcher: key => `value#${key}`, - }); - - await cache.fetch('foo'); - - expect(fakeInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); - }); - - it('should delete from memory cache', async () => { - const cache = makeCache(); - - await cache.set('foo', 'bar'); - await cache.delete('foo'); - - const result = cache.has('foo'); - expect(result).toBe(false); - }); - - it('should call onChanged when deleting', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - onChanged: fakeOnChanged, - }); - - await cache.set('foo', 'bar'); - await cache.delete('foo'); - - expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); - }); - - it('should emit event when deleting', async () => { - const cache = makeCache({ name: 'fake' }); - - await cache.set('foo', 'bar'); - await cache.delete('foo'); - - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); - }); - - it('should delete when receiving set event', async () => { - const cache = makeCache({ name: 'fake' }); - await cache.set('foo', 'bar'); - - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); - - const result = cache.has('foo'); - expect(result).toBe(false); - }); - - it('should call onChanged when receiving set event', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - onChanged: fakeOnChanged, - }); - - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); - - expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); - }); - - it('should delete when receiving delete event', async () => { - const cache = makeCache({ name: 'fake' }); - await cache.set('foo', 'bar'); - - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); - - const result = cache.has('foo'); - expect(result).toBe(false); - }); - - it('should call onChanged when receiving delete event', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - onChanged: fakeOnChanged, - }); - await cache.set('foo', 'bar'); - - await fakeInternalEventService._emitRedis('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); - - expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); - }); - - describe('get', () => { - it('should return value if present', async () => { - const cache = makeCache(); - await cache.set('foo', 'bar'); - - const result = cache.get('foo'); - - expect(result).toBe('bar'); - }); - it('should return undefined if missing', () => { - const cache = makeCache(); - - const result = cache.get('foo'); - - expect(result).toBe(undefined); - }); - }); - - describe('setMany', () => { - it('should populate all values', async () => { - const cache = makeCache(); - - await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); - - expect(cache.has('foo')).toBe(true); - expect(cache.has('alpha')).toBe(true); - }); - - it('should emit one event', async () => { - const cache = makeCache({ - name: 'fake', - }); - - await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); - - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); - }); - - it('should call onChanged once with all items', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - onChanged: fakeOnChanged, - }); - - await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); - - expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); - expect(fakeOnChanged).toHaveBeenCalledTimes(1); - }); - - it('should emit events only for changed items', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - onChanged: fakeOnChanged, - }); - - await cache.set('foo', 'bar'); - fakeOnChanged.mockClear(); - fakeInternalEventService._reset(); - - await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); - - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['alpha'] }]]); - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); - expect(fakeOnChanged).toHaveBeenCalledWith(['alpha'], cache); - expect(fakeOnChanged).toHaveBeenCalledTimes(1); - }); - }); - - describe('getMany', () => { - it('should return empty for empty input', () => { - const cache = makeCache(); - const result = cache.getMany([]); - expect(result).toEqual([]); - }); - - it('should return the value for all keys', () => { - const cache = makeCache(); - cache.add('foo', 'bar'); - cache.add('alpha', 'omega'); - - const result = cache.getMany(['foo', 'alpha']); - - expect(result).toEqual([['foo', 'bar'], ['alpha', 'omega']]); - }); - - it('should return undefined for missing keys', () => { - const cache = makeCache(); - cache.add('foo', 'bar'); - - const result = cache.getMany(['foo', 'alpha']); - - expect(result).toEqual([['foo', 'bar'], ['alpha', undefined]]); - }); - }); - - describe('fetchMany', () => { - it('should do nothing for empty input', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - onChanged: fakeOnChanged, - }); - - await cache.fetchMany([]); - - expect(fakeOnChanged).not.toHaveBeenCalled(); - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); - }); - - it('should return existing items', async () => { - const cache = makeCache(); - cache.add('foo', 'bar'); - cache.add('alpha', 'omega'); - - const result = await cache.fetchMany(['foo', 'alpha']); - - expect(result).toEqual([['foo', 'bar'], ['alpha', 'omega']]); - }); - - it('should return existing items without events', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - onChanged: fakeOnChanged, - }); - cache.add('foo', 'bar'); - cache.add('alpha', 'omega'); - - await cache.fetchMany(['foo', 'alpha']); - - expect(fakeOnChanged).not.toHaveBeenCalled(); - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); - }); - - it('should call bulkFetcher for missing items', async () => { - const cache = makeCache({ - bulkFetcher: keys => keys.map(k => [k, `${k}#many`]), - fetcher: key => `${key}#single`, - }); - - const results = await cache.fetchMany(['foo', 'alpha']); - - expect(results).toEqual([['foo', 'foo#many'], ['alpha', 'alpha#many']]); - }); - - it('should call bulkFetcher only once', async () => { - const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); - const cache = makeCache({ - bulkFetcher: mockBulkFetcher, - }); - - await cache.fetchMany(['foo', 'bar']); - - expect(mockBulkFetcher).toHaveBeenCalledTimes(1); - }); - - it('should call fetcher when fetchMany is undefined', async () => { - const cache = makeCache({ - fetcher: key => `${key}#single`, - }); - - const results = await cache.fetchMany(['foo', 'alpha']); - - expect(results).toEqual([['foo', 'foo#single'], ['alpha', 'alpha#single']]); - }); - - it('should call onChanged', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - onChanged: fakeOnChanged, - fetcher: k => k, - }); - - await cache.fetchMany(['foo', 'alpha']); - - expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); - expect(fakeOnChanged).toHaveBeenCalledTimes(1); - }); - - it('should call onChanged only for changed', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - onChanged: fakeOnChanged, - fetcher: k => k, - }); - cache.add('foo', 'bar'); - - await cache.fetchMany(['foo', 'alpha']); - - expect(fakeOnChanged).toHaveBeenCalledWith(['alpha'], cache); - expect(fakeOnChanged).toHaveBeenCalledTimes(1); - }); - - it('should not emit event', async () => { - const cache = makeCache({ - fetcher: k => k, - }); - - await cache.fetchMany(['foo', 'alpha']); - - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); - }); - }); - - describe('refreshMany', () => { - it('should do nothing for empty input', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - onChanged: fakeOnChanged, - }); - - const result = await cache.refreshMany([]); - - expect(result).toEqual([]); - expect(fakeOnChanged).not.toHaveBeenCalled(); - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); - }); - - it('should call bulkFetcher for all keys', async () => { - const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); - const cache = makeCache({ - bulkFetcher: mockBulkFetcher, - }); - - const result = await cache.refreshMany(['foo', 'alpha']); - - expect(result).toEqual([['foo', 'foo#value'], ['alpha', 'alpha#value']]); - expect(mockBulkFetcher).toHaveBeenCalledWith(['foo', 'alpha'], cache); - expect(mockBulkFetcher).toHaveBeenCalledTimes(1); - }); - - it('should replace any existing keys', async () => { - const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); - const cache = makeCache({ - bulkFetcher: mockBulkFetcher, - }); - cache.add('foo', 'bar'); - - const result = await cache.refreshMany(['foo', 'alpha']); - - expect(result).toEqual([['foo', 'foo#value'], ['alpha', 'alpha#value']]); - expect(mockBulkFetcher).toHaveBeenCalledWith(['foo', 'alpha'], cache); - expect(mockBulkFetcher).toHaveBeenCalledTimes(1); - }); - - it('should call onChanged for all keys', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - bulkFetcher: keys => keys.map(k => [k, `${k}#value`]), - onChanged: fakeOnChanged, - }); - cache.add('foo', 'bar'); - - await cache.refreshMany(['foo', 'alpha']); - - expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); - expect(fakeOnChanged).toHaveBeenCalledTimes(1); - }); - - it('should emit event for all keys', async () => { - const cache = makeCache({ - name: 'fake', - bulkFetcher: keys => keys.map(k => [k, `${k}#value`]), - }); - cache.add('foo', 'bar'); - - await cache.refreshMany(['foo', 'alpha']); - - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); - }); - }); - - describe('deleteMany', () => { - it('should remove keys from memory cache', async () => { - const cache = makeCache(); - - await cache.set('foo', 'bar'); - await cache.set('alpha', 'omega'); - await cache.deleteMany(['foo', 'alpha']); - - expect(cache.has('foo')).toBe(false); - expect(cache.has('alpha')).toBe(false); - }); - - it('should emit only one event', async () => { - const cache = makeCache({ - name: 'fake', - }); - - await cache.deleteMany(['foo', 'alpha']); - - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); - }); - - it('should call onChanged once with all items', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - onChanged: fakeOnChanged, - }); - - await cache.deleteMany(['foo', 'alpha']); - - expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], cache); - expect(fakeOnChanged).toHaveBeenCalledTimes(1); - }); - - it('should do nothing if no keys are provided', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - onChanged: fakeOnChanged, - }); - - await cache.deleteMany([]); - - expect(fakeOnChanged).not.toHaveBeenCalled(); - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); - }); - }); - - describe('refresh', () => { - it('should populate the value', async () => { - const cache = makeCache({ - name: 'fake', - fetcher: key => `value#${key}`, - }); - - await cache.refresh('foo'); - - const result = cache.has('foo'); - expect(result).toBe(true); - }); - - it('should return the value', async () => { - const cache = makeCache({ - name: 'fake', - fetcher: key => `value#${key}`, - }); - - const result = await cache.refresh('foo'); - - expect(result).toBe('value#foo'); - }); - - it('should replace the value if it exists', async () => { - const cache = makeCache({ - name: 'fake', - fetcher: key => `value#${key}`, - }); - - await cache.set('foo', 'bar'); - const result = await cache.refresh('foo'); - - expect(result).toBe('value#foo'); - }); - - it('should call onChanged', async () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - name: 'fake', - fetcher: key => `value#${key}`, - onChanged: fakeOnChanged, - }); - - await cache.refresh('foo'); - - expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], cache); - }); - - it('should emit event', async () => { - const cache = makeCache({ - name: 'fake', - fetcher: key => `value#${key}`, - }); - - await cache.refresh('foo'); - - expect(fakeInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); - }); - }); - - describe('add', () => { - it('should add the item', () => { - const cache = makeCache(); - cache.add('foo', 'bar'); - expect(cache.has('foo')).toBe(true); - }); - - it('should not emit event', () => { - const cache = makeCache({ - name: 'fake', - }); - - cache.add('foo', 'bar'); - - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); - }); - - it('should not call onChanged', () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - onChanged: fakeOnChanged, - }); - - cache.add('foo', 'bar'); - - expect(fakeOnChanged).not.toHaveBeenCalled(); - }); - }); - - describe('addMany', () => { - it('should add all items', () => { - const cache = makeCache(); - - cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); - - expect(cache.has('foo')).toBe(true); - expect(cache.has('alpha')).toBe(true); - }); - - it('should not emit event', () => { - const cache = makeCache({ - name: 'fake', - }); - - cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); - - expect(fakeInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); - }); - - it('should not call onChanged', () => { - const fakeOnChanged = jest.fn(() => Promise.resolve()); - const cache = makeCache({ - onChanged: fakeOnChanged, - }); - - cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); - - expect(fakeOnChanged).not.toHaveBeenCalled(); - }); - }); - - describe('has', () => { - it('should return false when empty', () => { - const cache = makeCache(); - const result = cache.has('foo'); - expect(result).toBe(false); - }); - - it('should return false when value is not in memory', async () => { - const cache = makeCache(); - await cache.set('foo', 'bar'); - - const result = cache.has('alpha'); - - expect(result).toBe(false); - }); - - it('should return true when value is in memory', async () => { - const cache = makeCache(); - await cache.set('foo', 'bar'); - - const result = cache.has('foo'); - - expect(result).toBe(true); - }); - }); - - describe('size', () => { - it('should return 0 when empty', () => { - const cache = makeCache(); - expect(cache.size).toBe(0); - }); - - it('should return correct size when populated', async () => { - const cache = makeCache(); - await cache.set('foo', 'bar'); - - expect(cache.size).toBe(1); - }); - }); - - describe('entries', () => { - it('should return empty when empty', () => { - const cache = makeCache(); - - const result = Array.from(cache.entries()); - - expect(result).toHaveLength(0); - }); - - it('should return all entries when populated', async () => { - const cache = makeCache(); - await cache.set('foo', 'bar'); - - const result = Array.from(cache.entries()); - - expect(result).toEqual([['foo', 'bar']]); - }); - }); - - describe('keys', () => { - it('should return empty when empty', () => { - const cache = makeCache(); - - const result = Array.from(cache.keys()); - - expect(result).toHaveLength(0); - }); - - it('should return all keys when populated', async () => { - const cache = makeCache(); - await cache.set('foo', 'bar'); - - const result = Array.from(cache.keys()); - - expect(result).toEqual(['foo']); - }); - }); - - describe('values', () => { - it('should return empty when empty', () => { - const cache = makeCache(); - - const result = Array.from(cache.values()); - - expect(result).toHaveLength(0); - }); - - it('should return all values when populated', async () => { - const cache = makeCache(); - await cache.set('foo', 'bar'); - - const result = Array.from(cache.values()); - - expect(result).toEqual(['bar']); - }); - }); - - describe('[Symbol.iterator]', () => { - it('should return empty when empty', () => { - const cache = makeCache(); - - const result = Array.from(cache); - - expect(result).toHaveLength(0); - }); - - it('should return all entries when populated', async () => { - const cache = makeCache(); - await cache.set('foo', 'bar'); - - const result = Array.from(cache); - - expect(result).toEqual([['foo', 'bar']]); - }); - }); -}); diff --git a/packages/backend/test/unit/misc/QuantumKVCacheTests.ts b/packages/backend/test/unit/misc/QuantumKVCacheTests.ts new file mode 100644 index 0000000000..a3d6abb3ad --- /dev/null +++ b/packages/backend/test/unit/misc/QuantumKVCacheTests.ts @@ -0,0 +1,1774 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { jest } from '@jest/globals'; +import { GodOfTimeService } from '../../misc/GodOfTimeService.js'; +import { MockInternalEventService } from '../../misc/MockInternalEventService.js'; +import * as assert from '../../misc/custom-assertions.js'; +import { QuantumKVCache, type QuantumKVOpts } from '@/misc/QuantumKVCache.js'; +import { KeyNotFoundError } from '@/misc/errors/KeyNotFoundError.js'; +import { FetchFailedError } from '@/misc/errors/FetchFailedError.js'; +import { DisposedError, DisposingError } from '@/misc/errors/DisposeError.js'; + +describe(QuantumKVCache, () => { + let mockTimeService: GodOfTimeService; + let mockInternalEventService: MockInternalEventService; + let madeCaches: QuantumKVCache[] = []; + + function makeCache(opts?: Partial> & { name?: string }): QuantumKVCache { + const _opts = { + name: expect.getState().currentTestName || 'test', + lifetime: Infinity, + fetcher: () => { throw new Error('not implemented'); }, + } satisfies QuantumKVOpts & { name: string }; + + if (opts) { + Object.assign(_opts, opts); + } + + const services = { + internalEventService: mockInternalEventService, + timeService: mockTimeService, + }; + + const cache = new QuantumKVCache(_opts.name, services, _opts); + madeCaches.push(cache); + return cache; + } + + beforeAll(() => { + mockTimeService = new GodOfTimeService(); + mockInternalEventService = new MockInternalEventService(); + }); + + afterEach(async () => { + for (const cache of madeCaches) { + await cache.dispose(); + } + madeCaches = []; + mockTimeService.reset(); + mockInternalEventService.mockReset(); + }); + + describe('dispose', () => { + it('should disconnect events', async () => { + const cache = makeCache(); + + await cache.dispose(); + + expect(mockInternalEventService._calls).toContainEqual(['off', ['quantumCacheUpdated', expect.anything()]]); + expect(mockInternalEventService._calls).toContainEqual(['off', ['quantumCacheReset', expect.anything()]]); + }); + + it('should clear memory cache', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + await cache.dispose(); + + expect(cache.size).toBe(0); + }); + + it('should prevent future calls', async () => { + const cache = makeCache(); + + await cache.dispose(); + + await assert.throwsAsync(DisposedError, async () => { + return await cache.set('foo', 'bar'); + }); + }); + + it('should pass dispose signal to fetchers', async () => { + let abortReason: unknown = undefined; + const cache = makeCache({ + fetcher: (key, meta) => { + meta.disposeSignal.addEventListener('abort', () => { + abortReason = meta.disposeSignal.reason; + }, { once: true }); + return `${key}#value`; + }, + }); + await cache.fetch('foo'); + + await cache.dispose(); + + expect(abortReason).toBeDefined(); + expect(abortReason).toBeInstanceOf(DisposingError); + }); + + it('should abort active fetches', async () => { + const testReady = Promise.withResolvers(); + const testComplete = Promise.withResolvers(); + const cache = makeCache({ + fetcher: async () => { + testReady.resolve(); + await testComplete.promise; + return 'test ending'; + }, + }); + const promise = cache.fetch('foo').finally(() => {}); + await testReady.promise; + + // must be in here: + await cache.dispose(); + + await assert.rejectsAsync(FetchFailedError, promise); + testComplete.resolve(); + }); + }); + + describe('set', () => { + it('should store in memory cache', async () => { + const cache = makeCache(); + + await cache.set('foo', 'bar'); + await cache.set('alpha', 'omega'); + + const result1 = cache.get('foo'); + expect(result1).toBe('bar'); + const result2 = cache.get('alpha'); + expect(result2).toBe('omega'); + }); + + it('should emit event when storing', async () => { + const cache = makeCache({ name: 'fake' }); + + await cache.set('foo', 'bar'); + + expect(mockInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); + }); + + it('should call onChanged when storing', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + await cache.set('foo', 'bar'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], expect.objectContaining({ cache })); + }); + + it('should not emit event when storing unchanged value', async () => { + const cache = makeCache(); + + await cache.set('foo', 'bar'); + await cache.set('foo', 'bar'); + + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + + it('should not call onChanged when storing unchanged value', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + await cache.set('foo', 'bar'); + await cache.set('foo', 'bar'); + + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + }); + + describe('constructor', () => { + it('should connect quantumCacheUpdated event', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + await cache.set('foo', 'foo'); + await cache.set('bar', 'bar'); + + await mockInternalEventService.mockEmit('quantumCacheUpdated', { name: 'fake', keys: ['foo'] }); + + expect(cache.size).toBe(1); + expect(cache.has('foo')).toBe(false); + expect(cache.has('bar')).toBe(true); + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], expect.objectContaining({ cache })); + expect(mockInternalEventService._calls).toContainEqual(['on', ['quantumCacheUpdated', expect.anything(), { ignoreLocal: true }]]); + }); + + it('should connect quantumCacheReset event', async () => { + const fakeOnReset = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onReset: fakeOnReset, + }); + await cache.set('foo', 'foo'); + await cache.set('bar', 'bar'); + + await mockInternalEventService.mockEmit('quantumCacheReset', { name: 'fake' }); + + expect(cache.size).toBe(0); + expect(fakeOnReset).toHaveBeenCalledWith(expect.objectContaining({ cache })); + expect(mockInternalEventService._calls).toContainEqual(['on', ['quantumCacheReset', expect.anything(), { ignoreLocal: true }]]); + }); + }); + + describe('get', () => { + it('should return value if present', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = cache.get('foo'); + + expect(result).toBe('bar'); + }); + + it('should throw KeyNotFoundError if missing', () => { + const cache = makeCache(); + + assert.throws(KeyNotFoundError, () => { + cache.get('foo'); + }); + }); + }); + + describe('getMaybe', () => { + it('should return value if present', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = cache.getMaybe('foo'); + + expect(result).toBe('bar'); + }); + + it('should return undefined if missing', () => { + const cache = makeCache(); + + const result = cache.getMaybe('foo'); + + expect(result).toBe(undefined); + }); + }); + + describe('setMany', () => { + it('should populate all values', async () => { + const cache = makeCache(); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(cache.has('foo')).toBe(true); + expect(cache.has('alpha')).toBe(true); + }); + + it('should emit one event', async () => { + const cache = makeCache({ + name: 'fake', + }); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(mockInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + + it('should call onChanged once with all items', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], expect.objectContaining({ cache })); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should emit events only for changed items', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + name: 'fake', + onChanged: fakeOnChanged, + }); + + await cache.set('foo', 'bar'); + fakeOnChanged.mockClear(); + mockInternalEventService.mockReset(); + + await cache.setMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(mockInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['alpha'] }]]); + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + expect(fakeOnChanged).toHaveBeenCalledWith(['alpha'], expect.objectContaining({ cache })); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + }); + + describe('getMany', () => { + it('should return empty for empty input', () => { + const cache = makeCache(); + const result = cache.getMany([]); + expect(result).toEqual([]); + }); + + it('should include the value of all found keys', () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + cache.add('alpha', 'omega'); + + const result = cache.getMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'bar'], ['alpha', 'omega']]); + }); + + it('should exclude all missing keys', () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + + const result = cache.getMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'bar']]); + }); + }); + + describe('fetch', () => { + it('should fetch an unknown value', async () => { + const cache = makeCache({ + fetcher: key => `value#${key}`, + }); + + const result = await cache.fetch('foo'); + + expect(result).toBe('value#foo'); + }); + + it('should store fetched value in memory cache', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.fetch('foo'); + + const result = cache.has('foo'); + expect(result).toBe(true); + }); + + it('should call onChanged', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + fetcher: key => `value#${key}`, + onChanged: fakeOnChanged, + }); + + await cache.fetch('foo'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], expect.objectContaining({ cache })); + }); + + it('should not emit event', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.fetch('foo'); + + expect(mockInternalEventService._calls).not.toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); + }); + + it('should throw FetchFailedError when fetcher throws error', async () => { + const cache = makeCache({ + fetcher: () => { throw new Error('test error'); }, + }); + + await assert.throwsAsync(FetchFailedError, async () => { + return await cache.fetch('foo'); + }); + }); + + it('should throw KeyNotFoundError when fetcher returns null', async () => { + const cache = makeCache({ + fetcher: () => null, + }); + + await assert.throwsAsync(KeyNotFoundError, async () => { + return await cache.fetch('foo'); + }); + }); + + it('should throw KeyNotFoundError when fetcher returns undefined', async () => { + const cache = makeCache({ + fetcher: () => undefined, + }); + + await assert.throwsAsync(KeyNotFoundError, async () => { + return await cache.fetch('foo'); + }); + }); + + it('should respect fetcherConcurrency', async () => { + await testConcurrency( + { + fetcher: key => `value#${key}`, + fetcherConcurrency: 2, + }, + (cache, key) => cache.fetch(key), + ['value#foo', 'value#bar', 'value#baz'], + ); + }); + + it('should respect maxConcurrency', async () => { + await testConcurrency( + { + fetcher: key => `value#${key}`, + maxConcurrency: 2, + }, + (cache, key) => cache.fetch(key), + ['value#foo', 'value#bar', 'value#baz'], + ); + }); + + it('should de-duplicate calls', async () => { + // Arrange + const testComplete = Promise.withResolvers(); + const mockFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const cache = makeCache({ fetcher: mockFetcher }); + + // Act + const fetch1 = cache.fetch('foo'); + const fetch2 = cache.fetch('foo'); + + // Assert + testComplete.resolve(); + await expect(fetch1).resolves.toBe('value#foo'); + await expect(fetch2).resolves.toBe('value#foo'); + expect(mockFetcher).toHaveBeenCalledTimes(1); + }); + }); + + describe('fetchMaybe', () => { + it('should return value when found by fetcher', async () => { + const cache = makeCache({ + optionalFetcher: () => 'bar', + }); + + const result = await cache.fetchMaybe('foo'); + + expect(result).toBe('bar'); + }); + + it('should persist value when found by fetcher', async () => { + const cache = makeCache({ + optionalFetcher: () => 'bar', + }); + + await cache.fetchMaybe('foo'); + const result = cache.get('foo'); + + expect(result).toBe('bar'); + }); + + it('should call onChanged when found by fetcher', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + optionalFetcher: () => 'bar', + onChanged: fakeOnChanged, + }); + + await cache.fetchMaybe('foo'); + + expect(fakeOnChanged).toHaveBeenCalled(); + }); + + it('should return undefined when fetcher returns undefined', async () => { + const cache = makeCache({ + optionalFetcher: () => undefined, + }); + + const result = await cache.fetchMaybe('foo'); + + expect(result).toBe(undefined); + }); + + it('should not call onChanged when fetcher returns undefined', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + optionalFetcher: () => undefined, + onChanged: fakeOnChanged, + }); + + await cache.fetchMaybe('foo'); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + }); + + it('should return undefined when fetcher returns null', async () => { + const cache = makeCache({ + optionalFetcher: () => null, + }); + + const result = await cache.fetchMaybe('foo'); + + expect(result).toBe(undefined); + }); + + it('should not call onChanged when fetcher returns null', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + optionalFetcher: () => null, + onChanged: fakeOnChanged, + }); + + await cache.fetchMaybe('foo'); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + }); + + it('should throw FetchFailedError when fetcher throws error', async () => { + const cache = makeCache({ + optionalFetcher: () => { throw new Error('test error'); }, + }); + + await assert.throwsAsync(FetchFailedError, async () => { + return await cache.fetchMaybe('foo'); + }); + }); + + it('should fall back on fetcher when optionalFetcher is not defined', async () => { + const cache = makeCache({ + fetcher: () => 'bar', + }); + + const result = await cache.fetchMaybe('foo'); + + expect(result).toBe('bar'); + }); + + it('should respect optionalFetcherConcurrency', async () => { + await testConcurrency( + { + optionalFetcher: key => `value#${key}`, + optionalFetcherConcurrency: 2, + }, + (cache, key) => cache.fetchMaybe(key), + ['value#foo', 'value#bar', 'value#baz'], + ); + }); + + it('should respect maxConcurrency', async () => { + await testConcurrency( + { + optionalFetcher: key => `value#${key}`, + maxConcurrency: 2, + }, + (cache, key) => cache.fetchMaybe(key), + ['value#foo', 'value#bar', 'value#baz'], + ); + }); + + it('should de-duplicate calls', async () => { + // Arrange + const testComplete = Promise.withResolvers(); + const mockFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const cache = makeCache({ optionalFetcher: mockFetcher }); + + // Act + const fetch1 = cache.fetchMaybe('foo'); + const fetch2 = cache.fetchMaybe('foo'); + + // Assert + testComplete.resolve(); + await expect(fetch1).resolves.toBe('value#foo'); + await expect(fetch2).resolves.toBe('value#foo'); + expect(mockFetcher).toHaveBeenCalledTimes(1); + }); + }); + + describe('fetchMany', () => { + it('should do nothing for empty input', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + await cache.fetchMany([]); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should return existing items', async () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + cache.add('alpha', 'omega'); + + const result = await cache.fetchMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'bar'], ['alpha', 'omega']]); + }); + + it('should return existing items without events', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + cache.add('foo', 'bar'); + cache.add('alpha', 'omega'); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should call bulkFetcher for missing items', async () => { + const cache = makeCache({ + bulkFetcher: keys => keys.map(k => [k, `${k}#many`]), + fetcher: key => `${key}#single`, + }); + + const results = await cache.fetchMany(['foo', 'alpha']); + + expect(results).toEqual([['foo', 'foo#many'], ['alpha', 'alpha#many']]); + }); + + it('should call bulkFetcher only once', async () => { + const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); + const cache = makeCache({ + bulkFetcher: mockBulkFetcher, + }); + + await cache.fetchMany(['foo', 'bar']); + + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + }); + + it('should call optionalFetcher for single item', async () => { + const cache = makeCache({ + optionalFetcher: () => 'good', + bulkFetcher: keys => keys.map(k => [k, 'bad']), + fetcher: () => 'bad', + }); + + const results = await cache.fetchMany(['foo']); + + expect(results).toEqual([['foo', 'good']]); + }); + + it('should call fetcher for single item when optionalFetcher is not defined', async () => { + const cache = makeCache({ + bulkFetcher: keys => keys.map(k => [k, 'bad']), + fetcher: () => 'good', + }); + + const results = await cache.fetchMany(['foo']); + + expect(results).toEqual([['foo', 'good']]); + }); + + it('should call fetcher when fetchMany is undefined', async () => { + const cache = makeCache({ + fetcher: key => `${key}#single`, + }); + + const results = await cache.fetchMany(['foo', 'alpha']); + + expect(results).toEqual([['foo', 'foo#single'], ['alpha', 'alpha#single']]); + }); + + it('should call onChanged', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + fetcher: k => k, + }); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], expect.objectContaining({ cache })); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should call onChanged only for changed', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + fetcher: k => k, + }); + cache.add('foo', 'bar'); + + await cache.fetchMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['alpha'], expect.objectContaining({ cache })); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should not emit event', async () => { + const cache = makeCache({ + fetcher: k => k, + }); + + await cache.fetchMany(['foo', 'alpha']); + + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should respect bulkFetcherConcurrency', async () => { + await testConcurrency( + { + bulkFetcher: keys => [[keys[0], `value#${keys[0]}`]], + bulkFetcherConcurrency: 2, + }, + (cache, key) => cache.fetchMany([key, `${key}#dupe`]), + [[['foo', 'value#foo']], [['bar', 'value#bar']], [['baz', 'value#baz']]], + ); + }); + + it('should respect maxConcurrency', async () => { + await testConcurrency( + { + bulkFetcher: keys => [[keys[0], `value#${keys[0]}`]], + maxConcurrency: 2, + }, + (cache, key) => cache.fetchMany([key, `${key}#dupe`]), + [[['foo', 'value#foo']], [['bar', 'value#bar']], [['baz', 'value#baz']]], + ); + }); + + it('should de-duplicate calls using fetcher', async () => { + // Arrange + const testComplete = Promise.withResolvers(); + const mockFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const mockBulkFetcher = jest.fn(async (keys: string[]) => { + await testComplete.promise; + return keys.map(key => [key, `value#${key}`] as [string, string]); + }); + const cache = makeCache({ + fetcher: mockFetcher, + bulkFetcher: mockBulkFetcher, + }); + + // Act + const fetch1 = cache.fetch('foo'); + const fetch2 = cache.fetchMany(['foo', 'bar', 'baz']); + + // Assert + testComplete.resolve(); + await expect(fetch1).resolves.toEqual('value#foo'); + await expect(fetch2).resolves.toEqual([['foo', 'value#foo'], ['bar', 'value#bar'], ['baz', 'value#baz']]); + expect(mockFetcher).toHaveBeenCalledTimes(1); + expect(mockFetcher).toHaveBeenCalledWith('foo', expect.objectContaining({ cache })); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + expect(mockBulkFetcher).toHaveBeenCalledWith(['bar', 'baz'], expect.objectContaining({ cache })); + }); + + it('should de-duplicate calls using optionalFetcher', async () => { + // Arrange + const testComplete = Promise.withResolvers(); + const mockFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const mockBulkFetcher = jest.fn(async (keys: string[]) => { + await testComplete.promise; + return keys.map(key => [key, `value#${key}`] as [string, string]); + }); + const cache = makeCache({ + optionalFetcher: mockFetcher, + bulkFetcher: mockBulkFetcher, + }); + + // Act + const fetch1 = cache.fetchMaybe('foo'); + const fetch2 = cache.fetchMany(['foo', 'bar', 'baz']); + + // Assert + testComplete.resolve(); + await expect(fetch1).resolves.toEqual('value#foo'); + await expect(fetch2).resolves.toEqual([['foo', 'value#foo'], ['bar', 'value#bar'], ['baz', 'value#baz']]); + expect(mockFetcher).toHaveBeenCalledTimes(1); + expect(mockFetcher).toHaveBeenCalledWith('foo', expect.objectContaining({ cache })); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + expect(mockBulkFetcher).toHaveBeenCalledWith(['bar', 'baz'], expect.objectContaining({ cache })); + }); + + it('should de-duplicate calls using fetcher and optionalFetcher', async () => { + // Arrange + const testComplete = Promise.withResolvers(); + const mockFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const mockOptionalFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const mockBulkFetcher = jest.fn(async (keys: string[]) => { + await testComplete.promise; + return keys.map(key => [key, `value#${key}`] as [string, string]); + }); + const cache = makeCache({ + fetcher: mockFetcher, + optionalFetcher: mockOptionalFetcher, + bulkFetcher: mockBulkFetcher, + }); + + // Act + const fetch1 = cache.fetch('foo'); + const fetch2 = cache.fetchMaybe('bar'); + const fetch3 = cache.fetchMany(['foo', 'bar', 'baz', 'wow']); + + // Assert + testComplete.resolve(); + await expect(fetch1).resolves.toEqual('value#foo'); + await expect(fetch2).resolves.toEqual('value#bar'); + await expect(fetch3).resolves.toEqual([['foo', 'value#foo'], ['bar', 'value#bar'], ['baz', 'value#baz'], ['wow', 'value#wow']]); + expect(mockFetcher).toHaveBeenCalledTimes(1); + expect(mockFetcher).toHaveBeenCalledWith('foo', expect.objectContaining({ cache })); + expect(mockOptionalFetcher).toHaveBeenCalledTimes(1); + expect(mockOptionalFetcher).toHaveBeenCalledWith('bar', expect.objectContaining({ cache })); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + expect(mockBulkFetcher).toHaveBeenCalledWith(['baz', 'wow'], expect.objectContaining({ cache })); + }); + }); + + describe('refresh', () => { + it('should populate the value', async () => { + const cache = makeCache({ + fetcher: key => `value#${key}`, + }); + + await cache.refresh('foo'); + + const result = cache.has('foo'); + expect(result).toBe(true); + }); + + it('should return the value', async () => { + const cache = makeCache({ + fetcher: key => `value#${key}`, + }); + + const result = await cache.refresh('foo'); + + expect(result).toBe('value#foo'); + }); + + it('should replace the value if it exists', async () => { + const cache = makeCache({ + fetcher: key => `value#${key}`, + }); + + await cache.set('foo', 'bar'); + const result = await cache.refresh('foo'); + + expect(result).toBe('value#foo'); + }); + + it('should call onChanged', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + fetcher: key => `value#${key}`, + onChanged: fakeOnChanged, + }); + + await cache.refresh('foo'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], expect.objectContaining({ cache })); + }); + + it('should emit event', async () => { + const cache = makeCache({ + name: 'fake', + fetcher: key => `value#${key}`, + }); + + await cache.refresh('foo'); + + expect(mockInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); + }); + + it('should respect fetcherConcurrency', async () => { + await testConcurrency( + { + fetcher: key => `value#${key}`, + fetcherConcurrency: 2, + }, + (cache, key) => cache.refresh(key), + ['value#foo', 'value#bar', 'value#baz'], + ); + }); + + it('should respect maxConcurrency', async () => { + await testConcurrency( + { + fetcher: key => `value#${key}`, + maxConcurrency: 2, + }, + (cache, key) => cache.refresh(key), + ['value#foo', 'value#bar', 'value#baz'], + ); + }); + + it('should de-duplicate calls', async () => { + // Arrange + const testComplete = Promise.withResolvers(); + const mockFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const cache = makeCache({ fetcher: mockFetcher }); + + // Act + const fetch1 = cache.refresh('foo'); + const fetch2 = cache.refresh('foo'); + + // Assert + testComplete.resolve(); + await expect(fetch1).resolves.toBe('value#foo'); + await expect(fetch2).resolves.toBe('value#foo'); + expect(mockFetcher).toHaveBeenCalledTimes(1); + }); + }); + + describe('refreshMaybe', () => { + it('should return value when found by fetcher', async () => { + const cache = makeCache({ + optionalFetcher: () => 'bar', + }); + + const result = await cache.refreshMaybe('foo'); + + expect(result).toBe('bar'); + }); + + it('should persist value when found by fetcher', async () => { + const cache = makeCache({ + optionalFetcher: () => 'bar', + }); + + await cache.refreshMaybe('foo'); + const result = cache.get('foo'); + + expect(result).toBe('bar'); + }); + + it('should call onChanged when found by fetcher', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + optionalFetcher: () => 'bar', + onChanged: fakeOnChanged, + }); + + await cache.refreshMaybe('foo'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], expect.objectContaining({ cache })); + }); + + it('should return undefined when fetcher returns undefined', async () => { + const cache = makeCache({ + optionalFetcher: () => undefined, + }); + + const result = await cache.refreshMaybe('foo'); + + expect(result).toBe(undefined); + }); + + it('should call onChanged when fetcher returns undefined', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + optionalFetcher: () => undefined, + onChanged: fakeOnChanged, + }); + + await cache.refreshMaybe('foo'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], expect.objectContaining({ cache })); + }); + + it('should return undefined when fetcher returns null', async () => { + const cache = makeCache({ + optionalFetcher: () => null, + }); + + const result = await cache.refreshMaybe('foo'); + + expect(result).toBe(undefined); + }); + + it('should call onChanged when fetcher returns null', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + optionalFetcher: () => null, + onChanged: fakeOnChanged, + }); + + await cache.refreshMaybe('foo'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], expect.objectContaining({ cache })); + }); + + it('should throw FetchFailedError when fetcher throws error', async () => { + const cache = makeCache({ + optionalFetcher: () => { throw new Error('test error'); }, + }); + + await assert.throwsAsync(FetchFailedError, async () => { + return await cache.refreshMaybe('foo'); + }); + }); + + it('should fall back on fetcher when optionalFetcher is not defined', async () => { + const cache = makeCache({ + fetcher: () => 'bar', + }); + + const result = await cache.refreshMaybe('foo'); + + expect(result).toBe('bar'); + }); + + it('should replace the value if it exists', async () => { + const cache = makeCache({ + optionalFetcher: key => `value#${key}`, + }); + + await cache.set('foo', 'bar'); + const result = await cache.refreshMaybe('foo'); + + expect(result).toBe('value#foo'); + }); + + it('should emit event when found', async () => { + const cache = makeCache({ + name: 'fake', + optionalFetcher: key => `value#${key}`, + }); + + await cache.refreshMaybe('foo'); + + expect(mockInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); + }); + + it('should emit event when not found', async () => { + const cache = makeCache({ + name: 'fake', + optionalFetcher: () => undefined, + }); + + await cache.refreshMaybe('foo'); + + expect(mockInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); + }); + + it('should respect optionalFetcherConcurrency', async () => { + await testConcurrency( + { + optionalFetcher: key => `value#${key}`, + optionalFetcherConcurrency: 2, + }, + (cache, key) => cache.refreshMaybe(key), + ['value#foo', 'value#bar', 'value#baz'], + ); + }); + + it('should respect maxConcurrency', async () => { + await testConcurrency( + { + fetcher: key => `value#${key}`, + maxConcurrency: 2, + }, + (cache, key) => cache.refreshMaybe(key), + ['value#foo', 'value#bar', 'value#baz'], + ); + }); + + it('should de-duplicate calls', async () => { + // Arrange + const testComplete = Promise.withResolvers(); + const mockFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const cache = makeCache({ optionalFetcher: mockFetcher }); + + // Act + const fetch1 = cache.refreshMaybe('foo'); + const fetch2 = cache.refreshMaybe('foo'); + + // Assert + testComplete.resolve(); + await expect(fetch1).resolves.toBe('value#foo'); + await expect(fetch2).resolves.toBe('value#foo'); + expect(mockFetcher).toHaveBeenCalledTimes(1); + }); + }); + + describe('refreshMany', () => { + it('should do nothing for empty input', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + const result = await cache.refreshMany([]); + + expect(result).toEqual([]); + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should call bulkFetcher for all keys', async () => { + const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); + const cache = makeCache({ + bulkFetcher: mockBulkFetcher, + }); + + const result = await cache.refreshMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'foo#value'], ['alpha', 'alpha#value']]); + expect(mockBulkFetcher).toHaveBeenCalledWith(['foo', 'alpha'], expect.objectContaining({ cache })); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + }); + + it('should replace any existing keys', async () => { + const mockBulkFetcher = jest.fn((keys: string[]) => keys.map(k => [k, `${k}#value`] as [string, string])); + const cache = makeCache({ + bulkFetcher: mockBulkFetcher, + }); + cache.add('foo', 'bar'); + + const result = await cache.refreshMany(['foo', 'alpha']); + + expect(result).toEqual([['foo', 'foo#value'], ['alpha', 'alpha#value']]); + expect(mockBulkFetcher).toHaveBeenCalledWith(['foo', 'alpha'], expect.objectContaining({ cache })); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + }); + + it('should call onChanged for all keys', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + bulkFetcher: keys => keys.map(k => [k, `${k}#value`]), + onChanged: fakeOnChanged, + }); + cache.add('foo', 'bar'); + + await cache.refreshMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], expect.objectContaining({ cache })); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should emit event for all keys', async () => { + const cache = makeCache({ + name: 'fake', + bulkFetcher: keys => keys.map(k => [k, `${k}#value`]), + }); + cache.add('foo', 'bar'); + + await cache.refreshMany(['foo', 'alpha']); + + expect(mockInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + + it('should call optionalFetcher for single item', async () => { + const cache = makeCache({ + optionalFetcher: () => 'good', + bulkFetcher: keys => keys.map(k => [k, 'bad']), + fetcher: () => 'bad', + }); + + const results = await cache.refreshMany(['foo']); + + expect(results).toEqual([['foo', 'good']]); + }); + + it('should call fetcher for single item when optionalFetcher is not defined', async () => { + const cache = makeCache({ + bulkFetcher: keys => keys.map(k => [k, 'bad']), + fetcher: () => 'good', + }); + + const results = await cache.refreshMany(['foo']); + + expect(results).toEqual([['foo', 'good']]); + }); + + it('should throw FetchFailedError when bulk fetcher throws error', async () => { + const cache = makeCache({ + bulkFetcher: () => { throw new Error('test error'); }, + }); + + await assert.throwsAsync(FetchFailedError, async () => { + return await cache.refreshMany(['foo']); + }); + }); + + it('should throw FetchFailedError when fallback fetcher throws error', async () => { + const cache = makeCache({ + fetcher: () => { throw new Error('test error'); }, + }); + + await assert.throwsAsync(FetchFailedError, async () => { + return await cache.refreshMany(['foo']); + }); + }); + + it('should not throw when fallback fetcher returns null', async () => { + const cache = makeCache({ + fetcher: () => null, + }); + + const result = await cache.refreshMany(['foo']); + + expect(result).toHaveLength(0); + }); + + it('should not throw when fallback fetcher returns undefined', async () => { + const cache = makeCache({ + fetcher: () => undefined, + }); + + const result = await cache.refreshMany(['foo']); + + expect(result).toHaveLength(0); + }); + + it('should not throw when bulk fetcher returns empty', async () => { + const cache = makeCache({ + bulkFetcher: () => [], + }); + + const result = await cache.refreshMany(['foo', 'bar']); + + expect(result).toHaveLength(0); + }); + + it('should respect bulkFetcherConcurrency', async () => { + await testConcurrency( + { + bulkFetcher: keys => [[keys[0], `value#${keys[0]}`]], + bulkFetcherConcurrency: 2, + }, + (cache, key) => cache.refreshMany([key, `${key}#dupe`]), + [[['foo', 'value#foo']], [['bar', 'value#bar']], [['baz', 'value#baz']]], + ); + }); + + it('should respect maxConcurrency', async () => { + await testConcurrency( + { + bulkFetcher: keys => [[keys[0], `value#${keys[0]}`]], + maxConcurrency: 2, + }, + (cache, key) => cache.refreshMany([key, `${key}#dupe`]), + [[['foo', 'value#foo']], [['bar', 'value#bar']], [['baz', 'value#baz']]], + ); + }); + + it('should de-duplicate calls using fetcher', async () => { + // Arrange + const testComplete = Promise.withResolvers(); + const mockFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const mockBulkFetcher = jest.fn(async (keys: string[]) => { + await testComplete.promise; + return keys.map(key => [key, `value#${key}`] as [string, string]); + }); + const cache = makeCache({ + fetcher: mockFetcher, + bulkFetcher: mockBulkFetcher, + }); + + // Act + const fetch1 = cache.fetch('foo'); + const fetch2 = cache.refreshMany(['foo', 'bar', 'baz']); + + // Assert + testComplete.resolve(); + await expect(fetch1).resolves.toEqual('value#foo'); + await expect(fetch2).resolves.toEqual([['foo', 'value#foo'], ['bar', 'value#bar'], ['baz', 'value#baz']]); + expect(mockFetcher).toHaveBeenCalledTimes(1); + expect(mockFetcher).toHaveBeenCalledWith('foo', expect.objectContaining({ cache })); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + expect(mockBulkFetcher).toHaveBeenCalledWith(['bar', 'baz'], expect.objectContaining({ cache })); + }); + + it('should de-duplicate calls using optionalFetcher', async () => { + // Arrange + const testComplete = Promise.withResolvers(); + const mockOptionalFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const mockBulkFetcher = jest.fn(async (keys: string[]) => { + await testComplete.promise; + return keys.map(key => [key, `value#${key}`] as [string, string]); + }); + const cache = makeCache({ + optionalFetcher: mockOptionalFetcher, + bulkFetcher: mockBulkFetcher, + }); + + // Act + const fetch1 = cache.fetchMaybe('foo'); + const fetch2 = cache.refreshMany(['foo', 'bar', 'baz']); + + // Assert + testComplete.resolve(); + await expect(fetch1).resolves.toEqual('value#foo'); + await expect(fetch2).resolves.toEqual([['foo', 'value#foo'], ['bar', 'value#bar'], ['baz', 'value#baz']]); + expect(mockOptionalFetcher).toHaveBeenCalledTimes(1); + expect(mockOptionalFetcher).toHaveBeenCalledWith('foo', expect.objectContaining({ cache })); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + expect(mockBulkFetcher).toHaveBeenCalledWith(['bar', 'baz'], expect.objectContaining({ cache })); + }); + + it('should de-duplicate calls using fetcher and optionalFetcher', async () => { + // Arrange + const testComplete = Promise.withResolvers(); + const mockFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const mockOptionalFetcher = jest.fn(async (key: string) => { + await testComplete.promise; + return `value#${key}`; + }); + const mockBulkFetcher = jest.fn(async (keys: string[]) => { + await testComplete.promise; + return keys.map(key => [key, `value#${key}`] as [string, string]); + }); + const cache = makeCache({ + fetcher: mockFetcher, + optionalFetcher: mockOptionalFetcher, + bulkFetcher: mockBulkFetcher, + }); + + // Act + const fetch1 = cache.fetch('foo'); + const fetch2 = cache.fetchMaybe('bar'); + const fetch3 = cache.refreshMany(['foo', 'bar', 'baz', 'wow']); + + // Assert + testComplete.resolve(); + await expect(fetch1).resolves.toEqual('value#foo'); + await expect(fetch2).resolves.toEqual('value#bar'); + await expect(fetch3).resolves.toEqual([['foo', 'value#foo'], ['bar', 'value#bar'], ['baz', 'value#baz'], ['wow', 'value#wow']]); + expect(mockFetcher).toHaveBeenCalledTimes(1); + expect(mockFetcher).toHaveBeenCalledWith('foo', expect.objectContaining({ cache })); + expect(mockOptionalFetcher).toHaveBeenCalledTimes(1); + expect(mockOptionalFetcher).toHaveBeenCalledWith('bar', expect.objectContaining({ cache })); + expect(mockBulkFetcher).toHaveBeenCalledTimes(1); + expect(mockBulkFetcher).toHaveBeenCalledWith(['baz', 'wow'], expect.objectContaining({ cache })); + }); + }); + + describe('delete', () => { + it('should delete from memory cache', async () => { + const cache = makeCache(); + + await cache.set('foo', 'bar'); + await cache.delete('foo'); + + const result = cache.has('foo'); + expect(result).toBe(false); + }); + + it('should call onChanged when deleting', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + await cache.set('foo', 'bar'); + await cache.delete('foo'); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo'], expect.objectContaining({ cache })); + }); + + it('should emit event when deleting', async () => { + const cache = makeCache({ name: 'fake' }); + + await cache.set('foo', 'bar'); + await cache.delete('foo'); + + expect(mockInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo'] }]]); + }); + }); + + describe('deleteMany', () => { + it('should remove keys from memory cache', async () => { + const cache = makeCache(); + + await cache.set('foo', 'bar'); + await cache.set('alpha', 'omega'); + await cache.deleteMany(['foo', 'alpha']); + + expect(cache.has('foo')).toBe(false); + expect(cache.has('alpha')).toBe(false); + }); + + it('should emit only one event', async () => { + const cache = makeCache({ + name: 'fake', + }); + + await cache.deleteMany(['foo', 'alpha']); + + expect(mockInternalEventService._calls).toContainEqual(['emit', ['quantumCacheUpdated', { name: 'fake', keys: ['foo', 'alpha'] }]]); + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(1); + }); + + it('should call onChanged once with all items', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + await cache.deleteMany(['foo', 'alpha']); + + expect(fakeOnChanged).toHaveBeenCalledWith(['foo', 'alpha'], expect.objectContaining({ cache })); + expect(fakeOnChanged).toHaveBeenCalledTimes(1); + }); + + it('should do nothing if no keys are provided', async () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + await cache.deleteMany([]); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + }); + + describe('reset', () => { + it('should erase all items', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + await cache.set('alpha', 'omega'); + + await cache.reset(); + + expect(cache.size).toBe(0); + }); + + it('should call onReset', async () => { + const fakeOnReset = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onReset: fakeOnReset, + }); + await cache.set('foo', 'bar'); + await cache.set('alpha', 'omega'); + + await cache.reset(); + + expect(fakeOnReset).toHaveBeenCalled(); + }); + + it('should emit event', async () => { + const cache = makeCache({ + name: 'fake', + }); + await cache.set('foo', 'bar'); + await cache.set('alpha', 'omega'); + + await cache.reset(); + + expect(mockInternalEventService._calls).toContainEqual(['emit', ['quantumCacheReset', { name: 'fake' }]]); + }); + }); + + describe('add', () => { + it('should add the item', () => { + const cache = makeCache(); + cache.add('foo', 'bar'); + expect(cache.has('foo')).toBe(true); + }); + + it('should not emit event', () => { + const cache = makeCache(); + + cache.add('foo', 'bar'); + + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should not call onChanged', () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + cache.add('foo', 'bar'); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + }); + }); + + describe('addMany', () => { + it('should add all items', () => { + const cache = makeCache(); + + cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(cache.has('foo')).toBe(true); + expect(cache.has('alpha')).toBe(true); + }); + + it('should not emit event', () => { + const cache = makeCache(); + + cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(mockInternalEventService._calls.filter(c => c[0] === 'emit')).toHaveLength(0); + }); + + it('should not call onChanged', () => { + const fakeOnChanged = jest.fn(() => Promise.resolve()); + const cache = makeCache({ + onChanged: fakeOnChanged, + }); + + cache.addMany([['foo', 'bar'], ['alpha', 'omega']]); + + expect(fakeOnChanged).not.toHaveBeenCalled(); + }); + }); + + describe('has', () => { + it('should return false when empty', () => { + const cache = makeCache(); + const result = cache.has('foo'); + expect(result).toBe(false); + }); + + it('should return false when value is not in memory', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = cache.has('alpha'); + + expect(result).toBe(false); + }); + + it('should return true when value is in memory', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = cache.has('foo'); + + expect(result).toBe(true); + }); + }); + + describe('size', () => { + it('should return 0 when empty', () => { + const cache = makeCache(); + expect(cache.size).toBe(0); + }); + + it('should return correct size when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + expect(cache.size).toBe(1); + }); + }); + + describe('entries', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache.entries()); + + expect(result).toHaveLength(0); + }); + + it('should return all entries when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache.entries()); + + expect(result).toEqual([['foo', 'bar']]); + }); + }); + + describe('keys', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache.keys()); + + expect(result).toHaveLength(0); + }); + + it('should return all keys when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache.keys()); + + expect(result).toEqual(['foo']); + }); + }); + + describe('values', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache.values()); + + expect(result).toHaveLength(0); + }); + + it('should return all values when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache.values()); + + expect(result).toEqual(['bar']); + }); + }); + + describe('[Symbol.iterator]', () => { + it('should return empty when empty', () => { + const cache = makeCache(); + + const result = Array.from(cache); + + expect(result).toHaveLength(0); + }); + + it('should return all entries when populated', async () => { + const cache = makeCache(); + await cache.set('foo', 'bar'); + + const result = Array.from(cache); + + expect(result).toEqual([['foo', 'bar']]); + }); + }); + + async function testConcurrency(opts: Partial>, fetchCallback: (cache: QuantumKVCache, key: string) => Promise, expectedResults: unknown): Promise { + const fetcher = opts.fetcher; + const optionalFetcher = opts.optionalFetcher; + const bulkFetcher = opts.bulkFetcher; + + // Arrange + const fetches = {} as Record>; + const testReady = Promise.withResolvers(); + const cache = makeCache({ + fetcherConcurrency: 4, + optionalFetcherConcurrency: 4, + bulkFetcherConcurrency: 4, + maxConcurrency: 4, + + ...opts, + + fetcher: fetcher ? async (key, meta) => { + await waitForSignalBeforeFetch(testReady, key, fetches); + return fetcher(key, meta); + } : undefined, + optionalFetcher: optionalFetcher ? async (key, meta) => { + await waitForSignalBeforeFetch(testReady, key, fetches); + return optionalFetcher(key, meta); + } : undefined, + bulkFetcher: bulkFetcher ? async (keys, meta) => { + await waitForSignalBeforeFetch(testReady, keys[0], fetches); + return bulkFetcher(keys, meta); + } : undefined, + }); + for (const key of ['foo', 'bar', 'baz']) { + const fetcher = { + created: false, + creating: Promise.withResolvers(), + gate: Promise.withResolvers(), + promise: fetchCallback(cache, key), + execute: async () => { + await fetcher.creating.promise; + return await fetcher.execute(); + }, + complete: async () => { + if (!fetcher.created) throw new Error(`test error: cannot complete an unstarted fetcher for ${key}`); + + fetcher.gate.resolve(); + return await fetcher.promise; + }, + }; + fetches[key] = fetcher; + } + + // Act + testReady.resolve(); + + // Assert: should create fetchers up to the limit + await Promise.all([fetches.foo.creating.promise, fetches.bar.creating.promise]); + expect(fetches.foo.created).toBe(true); + expect(fetches.bar.created).toBe(true); + expect(fetches.baz.created).toBe(false); + + // Assert: when one completes, should create the next one + await fetches.foo.complete(); + await fetches.baz.creating.promise; + expect(fetches.baz.created).toBe(true); + + // Assert: when all complete, final results should be correct + const results = await Promise.all([ + fetches.foo.complete(), + fetches.bar.complete(), + fetches.baz.complete(), + ]); + expect(results).toEqual(expectedResults); + } +}); + +// used for concurrency tests +async function waitForSignalBeforeFetch(testReady: PromiseWithResolvers, key: string, fetches: Record>) { + await testReady.promise; + + const fetch = fetches[key]; + expect(fetch).toBeTruthy(); + + fetch.created = true; + fetch.creating.resolve(); + + await fetch.gate.promise; +} + +// used for concurrency tests +interface FetchController { + // create phase + /** set to true when fetch callback is executed */ + created: boolean, + /** triggered internally when the callback is executed */ + creating: PromiseWithResolvers, + + // execute phase + /** triggered externally to start the fetcher */ + gate: PromiseWithResolvers, + /** resolves when fetcher completes */ + promise: Promise, + + // controls + /** starts and executes the fetcher */ + complete: () => Promise; + /** awaits creation, then starts and executes the fetcher */ + execute: () => Promise; +} diff --git a/packages/backend/test/unit/misc/call-all-tests.ts b/packages/backend/test/unit/misc/call-all-tests.ts new file mode 100644 index 0000000000..9d8ae0ec2c --- /dev/null +++ b/packages/backend/test/unit/misc/call-all-tests.ts @@ -0,0 +1,246 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { jest } from '@jest/globals'; +import * as assert from '../../misc/custom-assertions.js'; +import { callAll, callAllOn, callAllAsync, callAllOnAsync } from '@/misc/call-all.js'; + +describe(callAll, () => { + it('should call all functions when all succeed', () => { + const funcs = [ + jest.fn(() => {}), + jest.fn(() => {}), + jest.fn(() => {}), + ]; + + callAll(funcs); + + for (const func of funcs) { + expect(func).toHaveBeenCalledTimes(1); + } + }); + + it('should pass parameters to all functions', () => { + const funcs = [ + jest.fn((num: number) => expect(num).toBe(1)), + jest.fn((num: number) => expect(num).toBe(1)), + jest.fn((num: number) => expect(num).toBe(1)), + ]; + + callAll(funcs, 1); + }); + + it('should call all functions when some fail', () => { + const funcs = [ + jest.fn(() => { throw new Error(); }), + jest.fn(() => {}), + jest.fn(() => {}), + ]; + + try { + callAll(funcs); + } catch { + // ignore + } + + for (const func of funcs) { + expect(func).toHaveBeenCalledTimes(1); + } + }); + + it('should throw when some functions fail', () => { + const funcs = [ + jest.fn(() => { throw new Error(); }), + jest.fn(() => {}), + jest.fn(() => {}), + ]; + + assert.throws(AggregateError, () => { + callAll(funcs); + }); + }); + + it('should not throw when input is empty', () => { + expect(() => callAll([])).not.toThrow(); + }); +}); + +describe(callAllAsync, () => { + it('should call all functions when all succeed', async () => { + const funcs = [ + jest.fn(() => Promise.resolve()), + jest.fn(() => Promise.resolve()), + jest.fn(() => Promise.resolve()), + ]; + + await callAllAsync(funcs); + + for (const func of funcs) { + expect(func).toHaveBeenCalledTimes(1); + } + }); + + it('should pass parameters to all functions', async () => { + const funcs = [ + jest.fn((num: number) => expect(num).toBe(1)), + jest.fn((num: number) => expect(num).toBe(1)), + jest.fn((num: number) => expect(num).toBe(1)), + ]; + + await callAllAsync(funcs, 1); + }); + + it('should call all functions when some fail', async () => { + const funcs = [ + jest.fn(() => Promise.reject(new Error())), + jest.fn(() => Promise.resolve()), + jest.fn(() => Promise.resolve()), + ]; + + try { + await callAllAsync(funcs); + } catch { + // ignore + } + + for (const func of funcs) { + expect(func).toHaveBeenCalledTimes(1); + } + }); + + it('should throw when some functions fail', async () => { + const funcs = [ + jest.fn(() => Promise.reject(new Error())), + jest.fn(() => Promise.resolve()), + jest.fn(() => Promise.resolve()), + ]; + + await assert.throwsAsync(AggregateError, async () => { + await callAllAsync(funcs); + }); + }); + + it('should not throw when input is empty', async () => { + await callAllAsync([]); + }); +}); + +describe(callAllOn, () => { + it('should call all methods when all succeed', () => { + const objects = [ + { foo: jest.fn(() => {}) }, + { foo: jest.fn(() => {}) }, + { foo: jest.fn(() => {}) }, + ]; + + callAllOn(objects, 'foo'); + + for (const object of objects) { + expect(object.foo).toHaveBeenCalledTimes(1); + } + }); + + it('should pass parameters to all methods', () => { + const objects = [ + { foo: jest.fn((num: number) => expect(num).toBe(1)) }, + { foo: jest.fn((num: number) => expect(num).toBe(1)) }, + { foo: jest.fn((num: number) => expect(num).toBe(1)) }, + ]; + + callAllOn(objects, 'foo', 1); + }); + + it('should call all methods when some fail', () => { + const objects = [ + { foo: jest.fn(() => {}) }, + { foo: jest.fn(() => {}) }, + { foo: jest.fn(() => {}) }, + ]; + + try { + callAllOn(objects, 'foo'); + } catch { + // ignore + } + + for (const object of objects) { + expect(object.foo).toHaveBeenCalledTimes(1); + } + }); + + it('should throw when some methods fail', () => { + const objects = [ + { foo: jest.fn(() => { throw new Error(); }) }, + { foo: jest.fn(() => {}) }, + { foo: jest.fn(() => {}) }, + ]; + + expect(() => callAllOn(objects, 'foo')).toThrow(); + }); + + it('should not throw when input is empty', () => { + expect(() => callAllOn([] as { foo: () => void }[], 'foo')).not.toThrow(); + }); +}); + +describe(callAllOnAsync, () => { + it('should call all methods when all succeed', async () => { + const objects = [ + { foo: jest.fn(() => Promise.resolve()) }, + { foo: jest.fn(() => Promise.resolve()) }, + { foo: jest.fn(() => Promise.resolve()) }, + ]; + + await callAllOnAsync(objects, 'foo'); + + for (const object of objects) { + expect(object.foo).toHaveBeenCalledTimes(1); + } + }); + + it('should pass parameters to all methods', async () => { + const objects = [ + { foo: jest.fn((num: number) => expect(num).toBe(1)) }, + { foo: jest.fn((num: number) => expect(num).toBe(1)) }, + { foo: jest.fn((num: number) => expect(num).toBe(1)) }, + ]; + + await callAllOnAsync(objects, 'foo', 1); + }); + + it('should call all methods when some fail', async () => { + const objects = [ + { foo: jest.fn(() => Promise.resolve()) }, + { foo: jest.fn(() => Promise.resolve()) }, + { foo: jest.fn(() => Promise.resolve()) }, + ]; + + try { + await callAllOnAsync(objects, 'foo'); + } catch { + // ignore + } + + for (const object of objects) { + expect(object.foo).toHaveBeenCalledTimes(1); + } + }); + + it('should throw when some methods fail', async () => { + const objects = [ + { foo: jest.fn(() => Promise.reject(new Error())) }, + { foo: jest.fn(() => Promise.resolve()) }, + { foo: jest.fn(() => Promise.resolve()) }, + ]; + + await assert.throwsAsync(AggregateError, async () => { + await callAllOnAsync(objects, 'foo'); + }); + }); + + it('should not throw when input is empty', async () => { + await callAllOnAsync([] as { foo: () => void }[], 'foo'); + }); +}); diff --git a/packages/backend/test/unit/misc/is-retryable-error.ts b/packages/backend/test/unit/misc/is-retryable-error.ts index 6d241066f7..ac87eb5a0c 100644 --- a/packages/backend/test/unit/misc/is-retryable-error.ts +++ b/packages/backend/test/unit/misc/is-retryable-error.ts @@ -5,12 +5,12 @@ import { UnrecoverableError } from 'bullmq'; import { AbortError } from 'node-fetch'; -import { isRetryableError } from '@/misc/is-retryable-error.js'; +import { isRetryableError, isRetryableSymbol } from '@/misc/is-retryable-error.js'; import { StatusError } from '@/misc/status-error.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { CaptchaError, captchaErrorCodes } from '@/core/CaptchaService.js'; import { FastifyReplyError } from '@/misc/fastify-reply-error.js'; -import { ConflictError } from '@/server/SkRateLimiterService.js'; +import { CaptchaError, captchaErrorCodes } from '@/misc/captcha-error.js'; +import { ConflictError } from '@/misc/errors/ConflictError.js'; describe(isRetryableError, () => { it('should return true for retryable StatusError', () => { @@ -130,6 +130,54 @@ describe(isRetryableError, () => { expect(result).toBeFalsy(); }); + it('should return true for an object with [isRetryableSymbol]=true', () => { + const error = { + [isRetryableSymbol]: true, + }; + const result = isRetryableError(error); + expect(result).toBe(true); + }); + + it('should return true for an object with [isRetryableSymbol]=false', () => { + const error = { + [isRetryableSymbol]: false, + }; + const result = isRetryableError(error); + expect(result).toBe(false); + }); + + it('should return true for a retryable error with [isRetryableSymbol]=null', () => { + const error = Object.assign(new IdentifiableError('id', 'message', true), { + [isRetryableSymbol]: null, + }); + const result = isRetryableError(error); + expect(result).toBe(true); + }); + + it('should return false for a permanent error with [isRetryableSymbol]=null', () => { + const error = Object.assign(new IdentifiableError('id', 'message', false), { + [isRetryableSymbol]: null, + }); + const result = isRetryableError(error); + expect(result).toBe(false); + }); + + it('should return true for an ambiguous error with retryable cause', () => { + const error = new Error('error', { + cause: new IdentifiableError('id', 'cause', true), + }); + const result = isRetryableError(error); + expect(result).toBe(true); + }); + + it('should return false for an ambiguous error with permanent cause', () => { + const error = new Error('error', { + cause: new IdentifiableError('id', 'cause', false), + }); + const result = isRetryableError(error); + expect(result).toBe(false); + }); + const nonErrorInputs = [ [null, 'null'], [undefined, 'undefined'], @@ -138,6 +186,9 @@ describe(isRetryableError, () => { [true, 'boolean'], [[], 'array'], [{}, 'object'], + [{ [isRetryableSymbol]: null }, 'null isRetryableSymbol'], + [{ [isRetryableSymbol]: undefined }, 'undefined isRetryableSymbol'], + [{ [isRetryableSymbol]: '0' }, 'falsy isRetryableSymbol'], ]; for (const [input, label] of nonErrorInputs) { it(`should return true for ${label} input`, () => { diff --git a/packages/backend/test/unit/misc/kvp-array.ts b/packages/backend/test/unit/misc/kvp-array.ts new file mode 100644 index 0000000000..07a3943098 --- /dev/null +++ b/packages/backend/test/unit/misc/kvp-array.ts @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { makeKVPArray } from '@/misc/kvp-array.js'; + +describe(makeKVPArray, () => { + it('should add keys property', () => { + const array = [['1', 1], ['2', 2], ['3', 3]] as const; + + const result = makeKVPArray(array); + + expect(result).toHaveProperty('keys'); + }); + + it('should add values property', () => { + const array = [['1', 1], ['2', 2], ['3', 3]] as const; + + const result = makeKVPArray(array); + + expect(result).toHaveProperty('values'); + }); + + it('should preserve values', () => { + const array: [string, number][] = [['1', 1], ['2', 2], ['3', 3]]; + + const result = makeKVPArray(array); + + expect(result).toEqual(array); + }); + + it('should accept empty array', () => { + const array = [] as const; + + const result = makeKVPArray(array); + + expect(result).toHaveProperty('keys'); + expect(result).toHaveProperty('values'); + expect(result).toHaveLength(0); + }); +}); + +describe('keys', () => { + it('should return all keys', () => { + const array = [['1', 1], ['2', 2], ['3', 3]] as const; + + const result = makeKVPArray(array); + + expect(result.keys).toEqual(['1', '2', '3']); + }); + + it('should preserve duplicates', () => { + const array = [['1', 1], ['1', 1], ['1', 1]] as const; + + const result = makeKVPArray(array); + + expect(result.keys).toEqual(['1', '1', '1']); + }); +}); + +describe('values', () => { + it('should return all values', () => { + const array = [['1', 1], ['2', 2], ['3', 3]] as const; + + const result = makeKVPArray(array); + + expect(result.values).toEqual([1, 2, 3]); + }); + + it('should preserve duplicates', () => { + const array = [['1', 1], ['1', 1], ['1', 1]] as const; + + const result = makeKVPArray(array); + + expect(result.values).toEqual([1, 1, 1]); + }); +}); diff --git a/packages/backend/test/unit/queue/processors/CheckModeratorsActivityProcessorService.ts b/packages/backend/test/unit/queue/processors/CheckModeratorsActivityProcessorService.ts index 07618e7762..4f9dd9dafa 100644 --- a/packages/backend/test/unit/queue/processors/CheckModeratorsActivityProcessorService.ts +++ b/packages/backend/test/unit/queue/processors/CheckModeratorsActivityProcessorService.ts @@ -5,8 +5,9 @@ import { jest } from '@jest/globals'; import { Test, TestingModule } from '@nestjs/testing'; -import * as lolex from '@sinonjs/fake-timers'; import { addHours, addSeconds, subDays, subHours, subSeconds } from 'date-fns'; +import { GodOfTimeService } from '../../../misc/GodOfTimeService.js'; +import { MockConsole } from '../../../misc/MockConsole.js'; import { CheckModeratorsActivityProcessorService } from '@/queue/processors/CheckModeratorsActivityProcessorService.js'; import { MiSystemWebhook, MiUser, MiUserProfile, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; @@ -19,12 +20,16 @@ import { EmailService } from '@/core/EmailService.js'; import { SystemWebhookService } from '@/core/SystemWebhookService.js'; import { AnnouncementService } from '@/core/AnnouncementService.js'; import { SystemWebhookEventType } from '@/models/SystemWebhook.js'; +import { CacheManagementService } from '@/global/CacheManagementService.js'; +import { TimeService } from '@/global/TimeService.js'; +import { CoreModule } from '@/core/CoreModule.js'; +import { QueueProcessorModule } from '@/queue/QueueProcessorModule.js'; const baseDate = new Date(Date.UTC(2000, 11, 15, 12, 0, 0)); describe('CheckModeratorsActivityProcessorService', () => { let app: TestingModule; - let clock: lolex.InstalledClock; + let timeService: GodOfTimeService; let service: CheckModeratorsActivityProcessorService; // -------------------------------------------------------------------------------------- @@ -32,6 +37,7 @@ describe('CheckModeratorsActivityProcessorService', () => { let usersRepository: UsersRepository; let userProfilesRepository: UserProfilesRepository; let idService: IdService; + let cacheManagementService: CacheManagementService; let roleService: jest.Mocked; let announcementService: jest.Mocked; let emailService: jest.Mocked; @@ -89,61 +95,44 @@ describe('CheckModeratorsActivityProcessorService', () => { .createTestingModule({ imports: [ GlobalModule, - ], - providers: [ - CheckModeratorsActivityProcessorService, - IdService, - { - provide: RoleService, useFactory: () => ({ getModerators: jest.fn() }), - }, - { - provide: MetaService, useFactory: () => ({ fetch: jest.fn() }), - }, - { - provide: AnnouncementService, useFactory: () => ({ create: jest.fn() }), - }, - { - provide: EmailService, useFactory: () => ({ sendEmail: jest.fn() }), - }, - { - provide: SystemWebhookService, useFactory: () => ({ - fetchActiveSystemWebhooks: jest.fn(), - enqueueSystemWebhook: jest.fn(), - }), - }, - { - provide: QueueLoggerService, useFactory: () => ({ - logger: ({ - createSubLogger: () => ({ - info: jest.fn(), - warn: jest.fn(), - succ: jest.fn(), - }), - }), - }), - }, + CoreModule, + QueueProcessorModule, ], }) + .overrideProvider(TimeService).useClass(GodOfTimeService) + .overrideProvider(RoleService).useValue({ getModerators: jest.fn() }) + .overrideProvider(MetaService).useValue({ fetch: jest.fn() }) + .overrideProvider(AnnouncementService).useValue({ create: jest.fn() }) + .overrideProvider(EmailService).useValue({ sendEmail: jest.fn() }) + .overrideProvider(SystemWebhookService).useValue({ + fetchActiveSystemWebhooks: jest.fn(), + enqueueSystemWebhook: jest.fn(), + }) + .overrideProvider(DI.console).useClass(MockConsole) .compile(); + await app.init(); + app.enableShutdownHooks(); + usersRepository = app.get(DI.usersRepository); userProfilesRepository = app.get(DI.userProfilesRepository); service = app.get(CheckModeratorsActivityProcessorService); idService = app.get(IdService); + cacheManagementService = app.get(CacheManagementService); + timeService = app.get(TimeService); roleService = app.get(RoleService) as jest.Mocked; announcementService = app.get(AnnouncementService) as jest.Mocked; emailService = app.get(EmailService) as jest.Mocked; systemWebhookService = app.get(SystemWebhookService) as jest.Mocked; + }); - app.enableShutdownHooks(); + afterAll(async () => { + await app.close(); }); beforeEach(async () => { - clock = lolex.install({ - now: new Date(baseDate), - shouldClearNativeTimers: true, - }); + timeService.resetTo(baseDate.getTime()); systemWebhook1 = crateSystemWebhook({ on: ['inactiveModeratorsWarning'] }); systemWebhook2 = crateSystemWebhook({ on: ['inactiveModeratorsWarning', 'inactiveModeratorsInvitationOnlyChanged'] }); @@ -156,17 +145,13 @@ describe('CheckModeratorsActivityProcessorService', () => { }); afterEach(async () => { - clock.uninstall(); - await usersRepository.delete({}); - await userProfilesRepository.delete({}); + await usersRepository.deleteAll(); + await userProfilesRepository.deleteAll(); roleService.getModerators.mockReset(); announcementService.create.mockReset(); emailService.sendEmail.mockReset(); systemWebhookService.enqueueSystemWebhook.mockReset(); - }); - - afterAll(async () => { - await app.close(); + cacheManagementService.clear(); }); // -------------------------------------------------------------------------------------- diff --git a/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts b/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts index f7250600e3..c8e22c5e3c 100644 --- a/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts +++ b/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts @@ -3,79 +3,49 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import type Redis from 'ioredis'; +import { GodOfTimeService } from '../../../misc/GodOfTimeService.js'; +import { MockEnvService } from '../../../misc/MockEnvService.js'; +import { MockInternalEventService } from '../../../misc/MockInternalEventService.js'; +import { MockRedis } from '../../../misc/MockRedis.js'; import type { MiUser } from '@/models/User.js'; import type { RolePolicies, RoleService } from '@/core/RoleService.js'; +import type { Config } from '@/config.js'; import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { BucketRateLimit, Keyed, LegacyRateLimit } from '@/misc/rate-limit-utils.js'; - -/* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { CacheManagementService } from '@/global/CacheManagementService.js'; describe(SkRateLimiterService, () => { - let mockTimeService: { now: number, date: Date } = null!; - let mockRedis: Array<(command: [string, ...unknown[]]) => [Error | null, unknown] | null> = null!; - let mockRedisExec: (batch: [string, ...unknown[]][]) => Promise<[Error | null, unknown][] | null> = null!; - let mockEnvironment: Record = null!; - let serviceUnderTest: () => SkRateLimiterService = null!; - let mockDefaultUserPolicies: Partial = null!; - let mockUserPolicies: Record> = null!; + let cacheManagementService: CacheManagementService; + let mockInternalEventService: MockInternalEventService; + let mockTimeService: GodOfTimeService; + let mockRedis: MockRedis; + let mockEnvService: MockEnvService; + let serviceUnderTest: () => SkRateLimiterService; + let mockDefaultUserPolicies: Partial; + let mockUserPolicies: Record>; + let mockRoleService: RoleService; + + beforeAll(() => { + mockTimeService = new GodOfTimeService(); + mockEnvService = new MockEnvService(); + + mockRedis = new MockRedis(mockTimeService); + const fakeConfig = { host: 'example.com' } as unknown as Config; + mockInternalEventService = new MockInternalEventService(fakeConfig); + cacheManagementService = new CacheManagementService(mockRedis, mockTimeService, mockInternalEventService); + }); + + afterAll(() => { + cacheManagementService.dispose(); + mockInternalEventService.dispose(); + }); beforeEach(() => { - mockTimeService = { - now: 0, - get date() { - return new Date(mockTimeService.now); - }, - }; - - function callMockRedis(command: [string, ...unknown[]]) { - const handlerResults = mockRedis.map(handler => handler(command)); - const finalResult = handlerResults.findLast(result => result != null); - return finalResult ?? [null, null]; - } - - // I apologize to anyone who tries to read this later 🥲 - mockRedis = []; - mockRedisExec = (batch) => { - const results: [Error | null, unknown][] = batch.map(command => { - return callMockRedis(command); - }); - return Promise.resolve(results); - }; - const mockRedisClient = { - watch(...args: unknown[]) { - const result = callMockRedis(['watch', ...args]); - return Promise.resolve(result[0] ?? result[1]); - }, - get(...args: unknown[]) { - const result = callMockRedis(['get', ...args]); - return Promise.resolve(result[0] ?? result[1]); - }, - set(...args: unknown[]) { - const result = callMockRedis(['set', ...args]); - return Promise.resolve(result[0] ?? result[1]); - }, - multi(batch: [string, ...unknown[]][]) { - return { - exec() { - return mockRedisExec(batch); - }, - }; - }, - reset() { - return Promise.resolve(); - }, - } as unknown as Redis.Redis; - - mockEnvironment = Object.create(process.env); - mockEnvironment.NODE_ENV = 'production'; - const mockEnvService = { - env: mockEnvironment, - }; + mockTimeService.reset(); mockDefaultUserPolicies = { rateLimitFactor: 1 }; mockUserPolicies = {}; - const mockRoleService = { + mockRoleService = { getUserPolicies(key: string | null) { const policies = key != null ? mockUserPolicies[key] : null; return Promise.resolve(policies ?? mockDefaultUserPolicies); @@ -84,94 +54,44 @@ describe(SkRateLimiterService, () => { let service: SkRateLimiterService | undefined = undefined; serviceUnderTest = () => { - return service ??= new SkRateLimiterService(mockTimeService, mockRedisClient, mockRoleService, mockEnvService); + return service ??= new SkRateLimiterService(mockRedis, mockRoleService, mockTimeService, mockEnvService, cacheManagementService); }; }); + afterEach(() => { + cacheManagementService.dispose(); + mockInternalEventService.mockReset(); + mockRedis.mockReset(); + + mockEnvService.mockReset(); + mockEnvService.env.NODE_ENV = 'production'; + }); + describe('limit', () => { const actor = 'actor'; const key = 'test'; - let limitCounter: number | undefined = undefined; - let limitTimestamp: number | undefined = undefined; - - beforeEach(() => { - limitCounter = undefined; - limitTimestamp = undefined; - - mockRedis.push(([command, ...args]) => { - if (command === 'get') { - if (args[0] === 'rl_actor_test_c') { - const data = limitCounter?.toString() ?? null; - return [null, data]; - } - if (args[0] === 'rl_actor_test_t') { - const data = limitTimestamp?.toString() ?? null; - return [null, data]; - } - } - - if (command === 'set') { - if (args[0] === 'rl_actor_test_c') { - limitCounter = parseInt(args[1] as string); - return [null, args[1]]; - } - if (args[0] === 'rl_actor_test_t') { - limitTimestamp = parseInt(args[1] as string); - return [null, args[1]]; - } - } - - if (command === 'incr') { - if (args[0] === 'rl_actor_test_c') { - limitCounter = (limitCounter ?? 0) + 1; - return [null, null]; - } - if (args[0] === 'rl_actor_test_t') { - limitTimestamp = (limitTimestamp ?? 0) + 1; - return [null, null]; - } - } - - if (command === 'incrby') { - if (args[0] === 'rl_actor_test_c') { - limitCounter = (limitCounter ?? 0) + parseInt(args[1] as string); - return [null, null]; - } - if (args[0] === 'rl_actor_test_t') { - limitTimestamp = (limitTimestamp ?? 0) + parseInt(args[1] as string); - return [null, null]; - } - } - - if (command === 'decr') { - if (args[0] === 'rl_actor_test_c') { - limitCounter = (limitCounter ?? 0) - 1; - return [null, null]; - } - if (args[0] === 'rl_actor_test_t') { - limitTimestamp = (limitTimestamp ?? 0) - 1; - return [null, null]; - } - } - - if (command === 'decrby') { - if (args[0] === 'rl_actor_test_c') { - limitCounter = (limitCounter ?? 0) - parseInt(args[1] as string); - return [null, null]; - } - if (args[0] === 'rl_actor_test_t') { - limitTimestamp = (limitTimestamp ?? 0) - parseInt(args[1] as string); - return [null, null]; - } - } - - return null; - }); - }); + const limitCounter = { + get: async () => { + const c = await mockRedis.get('rl_actor_test_c'); + return c != null ? parseInt(c) : undefined; + }, + set: async (value: number) => { + await mockRedis.set('rl_actor_test_c', value); + }, + }; + const limitTimestamp = { + get: async () => { + const t = await mockRedis.get('rl_actor_test_t'); + return t != null ? parseInt(t) : undefined; + }, + set: async (value: number) => { + await mockRedis.set('rl_actor_test_t', value); + }, + }; it('should bypass in test environment', async () => { - mockEnvironment.NODE_ENV = 'test'; + mockEnvService.env.NODE_ENV = 'test'; const info = await serviceUnderTest().limit({ key: 'l', type: undefined, max: 0 }, actor); @@ -184,7 +104,7 @@ describe(SkRateLimiterService, () => { }); describe('with bucket limit', () => { - let limit: Keyed = null!; + let limit: Keyed; beforeEach(() => { limit = { @@ -202,8 +122,8 @@ describe(SkRateLimiterService, () => { it('should return correct info when allowed', async () => { limit.size = 2; - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); const info = await serviceUnderTest().limit(limit, actor); @@ -217,7 +137,7 @@ describe(SkRateLimiterService, () => { it('should increment counter when called', async () => { await serviceUnderTest().limit(limit, actor); - expect(limitCounter).toBe(1); + expect(await limitCounter.get()).toBe(1); }); it('should set timestamp when called', async () => { @@ -225,28 +145,28 @@ describe(SkRateLimiterService, () => { await serviceUnderTest().limit(limit, actor); - expect(limitTimestamp).toBe(1000); + expect(await limitTimestamp.get()).toBe(1000); }); it('should decrement counter when dripRate has passed', async () => { - limitCounter = 2; - limitTimestamp = 0; + await limitCounter.set(2); + await limitTimestamp.set(0); mockTimeService.now = 2000; await serviceUnderTest().limit(limit, actor); - expect(limitCounter).toBe(1); // 2 (starting) - 2 (2x1 drip) + 1 (call) = 1 + expect(await limitCounter.get()).toBe(1); // 2 (starting) - 2 (2x1 drip) + 1 (call) = 1 }); it('should decrement counter by dripSize', async () => { - limitCounter = 2; - limitTimestamp = 0; + await limitCounter.set(2); + await limitTimestamp.set(0); limit.dripSize = 2; mockTimeService.now = 1000; await serviceUnderTest().limit(limit, actor); - expect(limitCounter).toBe(1); // 2 (starting) - 2 (1x2 drip) + 1 (call) = 1 + expect(await limitCounter.get()).toBe(1); // 2 (starting) - 2 (1x2 drip) + 1 (call) = 1 }); it('should maintain counter between calls over time', async () => { @@ -261,13 +181,13 @@ describe(SkRateLimiterService, () => { mockTimeService.now += 1000; // 2 - 1 = 1 await serviceUnderTest().limit(limit, actor); // 1 + 1 = 2 - expect(limitCounter).toBe(2); - expect(limitTimestamp).toBe(3000); + expect(await limitCounter.get()).toBe(2); + expect(await limitTimestamp.get()).toBe(3000); }); it('should block when bucket is filled', async () => { - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); const info = await serviceUnderTest().limit(limit, actor); @@ -275,8 +195,8 @@ describe(SkRateLimiterService, () => { }); it('should calculate correct info when blocked', async () => { - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); const info = await serviceUnderTest().limit(limit, actor); @@ -287,8 +207,8 @@ describe(SkRateLimiterService, () => { }); it('should allow when bucket is filled but should drip', async () => { - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); mockTimeService.now = 1000; const info = await serviceUnderTest().limit(limit, actor); @@ -298,8 +218,8 @@ describe(SkRateLimiterService, () => { it('should scale limit by factor', async () => { mockDefaultUserPolicies.rateLimitFactor = 0.5; - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); const i1 = await serviceUnderTest().limit(limit, actor); // 1 + 1 = 2 const i2 = await serviceUnderTest().limit(limit, actor); // 2 + 1 = 3 @@ -312,38 +232,28 @@ describe(SkRateLimiterService, () => { }); it('should set counter expiration', async () => { - const commands: unknown[][] = []; - mockRedis.push(command => { - commands.push(command); - return null; - }); - await serviceUnderTest().limit(limit, actor); + mockTimeService.tick(1000); - expect(commands).toContainEqual(['expire', 'rl_actor_test_c', 1]); + expect(await limitCounter.get()).toBe(undefined); }); it('should set timestamp expiration', async () => { - const commands: unknown[][] = []; - mockRedis.push(command => { - commands.push(command); - return null; - }); - await serviceUnderTest().limit(limit, actor); + mockTimeService.tick(1000); - expect(commands).toContainEqual(['expire', 'rl_actor_test_t', 1]); + expect(await limitTimestamp.get()).toBe(undefined); }); it('should not increment when already blocked', async () => { - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); mockTimeService.now += 100; await serviceUnderTest().limit(limit, actor); - expect(limitCounter).toBe(1); - expect(limitTimestamp).toBe(0); + expect(await limitCounter.get()).toBe(1); + expect(await limitTimestamp.get()).toBe(0); }); it('should skip if factor is zero', async () => { @@ -436,7 +346,7 @@ describe(SkRateLimiterService, () => { }); it('should apply correction if extra calls slip through', async () => { - limitCounter = 2; + await limitCounter.set(2); const info = await serviceUnderTest().limit(limit, actor); @@ -451,8 +361,8 @@ describe(SkRateLimiterService, () => { it('should look up factor by user ID', async () => { const userActor = { id: actor } as unknown as MiUser; mockUserPolicies[actor] = { rateLimitFactor: 0.5 }; - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); const i1 = await serviceUnderTest().limit(limit, userActor); // 1 + 1 = 2 const i2 = await serviceUnderTest().limit(limit, userActor); // 2 + 1 = 3 @@ -463,7 +373,7 @@ describe(SkRateLimiterService, () => { }); describe('with min interval', () => { - let limit: Keyed = null!; + let limit: Keyed; beforeEach(() => { limit = { @@ -492,8 +402,8 @@ describe(SkRateLimiterService, () => { it('should increment counter when called', async () => { await serviceUnderTest().limit(limit, actor); - expect(limitCounter).not.toBeUndefined(); - expect(limitCounter).toBe(1); + expect(await limitCounter.get()).not.toBeUndefined(); + expect(await limitCounter.get()).toBe(1); }); it('should set timestamp when called', async () => { @@ -501,19 +411,19 @@ describe(SkRateLimiterService, () => { await serviceUnderTest().limit(limit, actor); - expect(limitCounter).not.toBeUndefined(); - expect(limitTimestamp).toBe(1000); + expect(await limitCounter.get()).not.toBeUndefined(); + expect(await limitTimestamp.get()).toBe(1000); }); it('should decrement counter when minInterval has passed', async () => { - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); mockTimeService.now = 1000; await serviceUnderTest().limit(limit, actor); - expect(limitCounter).not.toBeUndefined(); - expect(limitCounter).toBe(1); // 1 (starting) - 1 (interval) + 1 (call) = 1 + expect(await limitCounter.get()).not.toBeUndefined(); + expect(await limitCounter.get()).toBe(1); // 1 (starting) - 1 (interval) + 1 (call) = 1 }); it('should maintain counter between calls over time', async () => { @@ -527,13 +437,13 @@ describe(SkRateLimiterService, () => { const info = await serviceUnderTest().limit(limit, actor); // 0 + 1 = 1 expect(info.blocked).toBeFalsy(); - expect(limitCounter).toBe(1); - expect(limitTimestamp).toBe(3000); + expect(await limitCounter.get()).toBe(1); + expect(await limitTimestamp.get()).toBe(3000); }); it('should block when interval exceeded', async () => { - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); const info = await serviceUnderTest().limit(limit, actor); @@ -541,8 +451,8 @@ describe(SkRateLimiterService, () => { }); it('should calculate correct info when blocked', async () => { - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); const info = await serviceUnderTest().limit(limit, actor); @@ -553,8 +463,8 @@ describe(SkRateLimiterService, () => { }); it('should allow when bucket is filled but interval has passed', async () => { - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); mockTimeService.now = 1000; const info = await serviceUnderTest().limit(limit, actor); @@ -564,8 +474,8 @@ describe(SkRateLimiterService, () => { it('should scale interval by factor', async () => { mockDefaultUserPolicies.rateLimitFactor = 0.5; - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); const i1 = await serviceUnderTest().limit(limit, actor); const i2 = await serviceUnderTest().limit(limit, actor); @@ -578,38 +488,28 @@ describe(SkRateLimiterService, () => { }); it('should set counter expiration', async () => { - const commands: unknown[][] = []; - mockRedis.push(command => { - commands.push(command); - return null; - }); - await serviceUnderTest().limit(limit, actor); + mockTimeService.tick(1000); - expect(commands).toContainEqual(['expire', 'rl_actor_test_c', 1]); + expect(await limitCounter.get()).toBe(undefined); }); - it('should set timer expiration', async () => { - const commands: unknown[][] = []; - mockRedis.push(command => { - commands.push(command); - return null; - }); - + it('should set timestamp expiration', async () => { await serviceUnderTest().limit(limit, actor); + mockTimeService.tick(1000); - expect(commands).toContainEqual(['expire', 'rl_actor_test_t', 1]); + expect(await limitTimestamp.get()).toBe(undefined); }); it('should not increment when already blocked', async () => { - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); mockTimeService.now += 100; await serviceUnderTest().limit(limit, actor); - expect(limitCounter).toBe(1); - expect(limitTimestamp).toBe(0); + expect(await limitCounter.get()).toBe(1); + expect(await limitTimestamp.get()).toBe(0); }); it('should skip if factor is zero', async () => { @@ -647,7 +547,7 @@ describe(SkRateLimiterService, () => { }); it('should apply correction if extra calls slip through', async () => { - limitCounter = 2; + await limitCounter.set(2); const info = await serviceUnderTest().limit(limit, actor); @@ -661,7 +561,7 @@ describe(SkRateLimiterService, () => { }); describe('with legacy limit', () => { - let limit: Keyed = null!; + let limit: Keyed; beforeEach(() => { limit = { @@ -681,8 +581,8 @@ describe(SkRateLimiterService, () => { it('should infer dripRate from duration', async () => { limit.max = 10; limit.duration = 10000; - limitCounter = 10; - limitTimestamp = 0; + await limitCounter.set(10); + await limitTimestamp.set(0); const i1 = await serviceUnderTest().limit(limit, actor); mockTimeService.now += 1000; @@ -705,8 +605,8 @@ describe(SkRateLimiterService, () => { it('should calculate correct info when allowed', async () => { limit.max = 10; limit.duration = 10000; - limitCounter = 10; - limitTimestamp = 0; + await limitCounter.set(10); + await limitTimestamp.set(0); mockTimeService.now += 2000; const info = await serviceUnderTest().limit(limit, actor); @@ -721,8 +621,8 @@ describe(SkRateLimiterService, () => { it('should calculate correct info when blocked', async () => { limit.max = 10; limit.duration = 10000; - limitCounter = 10; - limitTimestamp = 0; + await limitCounter.set(10); + await limitTimestamp.set(0); const info = await serviceUnderTest().limit(limit, actor); @@ -734,8 +634,8 @@ describe(SkRateLimiterService, () => { }); it('should allow when bucket is filled but interval has passed', async () => { - limitCounter = 10; - limitTimestamp = 0; + await limitCounter.set(10); + await limitTimestamp.set(0); mockTimeService.now = 1000; const info = await serviceUnderTest().limit(limit, actor); @@ -745,8 +645,8 @@ describe(SkRateLimiterService, () => { it('should scale limit by factor', async () => { mockDefaultUserPolicies.rateLimitFactor = 0.5; - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); const i1 = await serviceUnderTest().limit(limit, actor); const i2 = await serviceUnderTest().limit(limit, actor); @@ -759,46 +659,36 @@ describe(SkRateLimiterService, () => { }); it('should set counter expiration', async () => { - const commands: unknown[][] = []; - mockRedis.push(command => { - commands.push(command); - return null; - }); - await serviceUnderTest().limit(limit, actor); + mockTimeService.tick(1000); - expect(commands).toContainEqual(['expire', 'rl_actor_test_c', 1]); + expect(await limitCounter.get()).toBe(undefined); }); it('should set timestamp expiration', async () => { - const commands: unknown[][] = []; - mockRedis.push(command => { - commands.push(command); - return null; - }); - await serviceUnderTest().limit(limit, actor); + mockTimeService.tick(1000); - expect(commands).toContainEqual(['expire', 'rl_actor_test_t', 1]); + expect(await limitTimestamp.get()).toBe(undefined); }); it('should not increment when already blocked', async () => { - limitCounter = 1; - limitTimestamp = 0; + await limitCounter.set(1); + await limitTimestamp.set(0); mockTimeService.now += 100; await serviceUnderTest().limit(limit, actor); - expect(limitCounter).toBe(1); - expect(limitTimestamp).toBe(0); + expect(await limitCounter.get()).toBe(1); + expect(await limitTimestamp.get()).toBe(0); }); it('should not allow dripRate to be lower than 0', async () => { // real-world case; taken from StreamingApiServerService limit.max = 4096; limit.duration = 2000; - limitCounter = 4096; - limitTimestamp = 0; + await limitCounter.set(4096); + await limitTimestamp.set(0); const i1 = await serviceUnderTest().limit(limit, actor); mockTimeService.now = 1; @@ -851,7 +741,7 @@ describe(SkRateLimiterService, () => { }); it('should apply correction if extra calls slip through', async () => { - limitCounter = 2; + await limitCounter.set(2); const info = await serviceUnderTest().limit(limit, actor); @@ -865,7 +755,7 @@ describe(SkRateLimiterService, () => { }); describe('with legacy limit and min interval', () => { - let limit: Keyed = null!; + let limit: Keyed; beforeEach(() => { limit = { @@ -884,8 +774,8 @@ describe(SkRateLimiterService, () => { }); it('should block when limit exceeded', async () => { - limitCounter = 10; - limitTimestamp = 0; + await limitCounter.set(10); + await limitTimestamp.set(0); const info = await serviceUnderTest().limit(limit, actor); @@ -893,8 +783,8 @@ describe(SkRateLimiterService, () => { }); it('should calculate correct info when allowed', async () => { - limitCounter = 9; - limitTimestamp = 0; + await limitCounter.set(9); + await limitTimestamp.set(0); const info = await serviceUnderTest().limit(limit, actor); @@ -906,8 +796,8 @@ describe(SkRateLimiterService, () => { }); it('should calculate correct info when blocked', async () => { - limitCounter = 10; - limitTimestamp = 0; + await limitCounter.set(10); + await limitTimestamp.set(0); const info = await serviceUnderTest().limit(limit, actor); @@ -919,8 +809,8 @@ describe(SkRateLimiterService, () => { }); it('should allow when counter is filled but interval has passed', async () => { - limitCounter = 5; - limitTimestamp = 0; + await limitCounter.set(5); + await limitTimestamp.set(0); mockTimeService.now = 1000; const info = await serviceUnderTest().limit(limit, actor); @@ -929,8 +819,8 @@ describe(SkRateLimiterService, () => { }); it('should drip according to minInterval', async () => { - limitCounter = 10; - limitTimestamp = 0; + await limitCounter.set(10); + await limitTimestamp.set(0); mockTimeService.now += 1000; const i1 = await serviceUnderTest().limit(limit, actor); @@ -944,8 +834,8 @@ describe(SkRateLimiterService, () => { it('should scale limit and interval by factor', async () => { mockDefaultUserPolicies.rateLimitFactor = 0.5; - limitCounter = 19; - limitTimestamp = 0; + await limitCounter.set(19); + await limitTimestamp.set(0); const i1 = await serviceUnderTest().limit(limit, actor); const i2 = await serviceUnderTest().limit(limit, actor); @@ -958,42 +848,32 @@ describe(SkRateLimiterService, () => { }); it('should set counter expiration', async () => { - const commands: unknown[][] = []; - mockRedis.push(command => { - commands.push(command); - return null; - }); - await serviceUnderTest().limit(limit, actor); + mockTimeService.tick(5000); - expect(commands).toContainEqual(['expire', 'rl_actor_test_c', 5]); + expect(await limitCounter.get()).toBe(undefined); }); it('should set timestamp expiration', async () => { - const commands: unknown[][] = []; - mockRedis.push(command => { - commands.push(command); - return null; - }); - await serviceUnderTest().limit(limit, actor); + mockTimeService.tick(5000); - expect(commands).toContainEqual(['expire', 'rl_actor_test_t', 5]); + expect(await limitTimestamp.get()).toBe(undefined); }); it('should not increment when already blocked', async () => { - limitCounter = 10; - limitTimestamp = 0; + await limitCounter.set(10); + await limitTimestamp.set(0); mockTimeService.now += 100; await serviceUnderTest().limit(limit, actor); - expect(limitCounter).toBe(10); - expect(limitTimestamp).toBe(0); + expect(await limitCounter.get()).toBe(10); + expect(await limitTimestamp.get()).toBe(0); }); it('should apply correction if extra calls slip through', async () => { - limitCounter = 12; + await limitCounter.set(12); const info = await serviceUnderTest().limit(limit, actor); diff --git a/packages/backend/test/unit/server/api/drive/files/create.ts b/packages/backend/test/unit/server/api/drive/files/create.ts index 9b38f4d744..9cfe0ea0f0 100644 --- a/packages/backend/test/unit/server/api/drive/files/create.ts +++ b/packages/backend/test/unit/server/api/drive/files/create.ts @@ -32,6 +32,8 @@ describe('/drive/files/create', () => { module = await Test.createTestingModule({ imports: [GlobalModule, CoreModule, ServerModule], }).compile(); + + await module.init(); module.enableShutdownHooks(); const serverService = module.get(ServerService); @@ -41,7 +43,7 @@ describe('/drive/files/create', () => { idService = module.get(IdService); const usersRepository = module.get(DI.usersRepository); - await usersRepository.delete({}); + await usersRepository.deleteAll(); root = await usersRepository.insert({ id: idService.gen(), username: 'root', @@ -50,7 +52,7 @@ describe('/drive/files/create', () => { }).then(x => usersRepository.findOneByOrFail(x.identifiers[0])); const userProfilesRepository = module.get(DI.userProfilesRepository); - await userProfilesRepository.delete({}); + await userProfilesRepository.deleteAll(); await userProfilesRepository.insert({ userId: root.id, }); diff --git a/packages/backend/src/server/api/endpoints/notes/create.test.ts b/packages/backend/test/unit/server/api/endpoints/notes/create.test.ts similarity index 92% rename from packages/backend/src/server/api/endpoints/notes/create.test.ts rename to packages/backend/test/unit/server/api/endpoints/notes/create.test.ts index 545889a7ee..5d333a1dab 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.test.ts +++ b/packages/backend/test/unit/server/api/endpoints/notes/create.test.ts @@ -6,11 +6,16 @@ process.env.NODE_ENV = 'test'; import { describe, test, expect } from '@jest/globals'; +import { MockConsole } from '../../../../../misc/MockConsole.js'; +import { getValidator } from '../../../../../../test/prelude/get-api-validator.js'; import { loadConfig } from '@/config.js'; -import { getValidator } from '../../../../../test/prelude/get-api-validator.js'; -import { paramDef } from './create.js'; +import { paramDef } from '@/server/api/endpoints/notes/create.js'; +import { NativeTimeService } from '@/global/TimeService.js'; +import { EnvService } from '@/global/EnvService.js'; +import { LoggerService } from '@/core/LoggerService.js'; -const config = loadConfig(); +const loggerService = new LoggerService(new MockConsole, new NativeTimeService(), new EnvService()); +const config = loadConfig(loggerService); const VALID = true; const INVALID = false; diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 25a3e07f9f..4a0d73a8b9 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -21,6 +21,9 @@ import type * as misskey from 'misskey-js'; import { DEFAULT_POLICIES } from '@/core/RoleService.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; import { ApiError } from '@/server/api/error.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import { EnvService } from '@/global/EnvService.js'; +import { NativeTimeService } from '@/global/TimeService.js'; export { server as startServer, jobQueue as startJobQueue } from '@/boot/common.js'; @@ -38,7 +41,9 @@ export type SystemWebhookPayload = { body: any; }; -const config = loadConfig(); +// eslint-disable-next-line no-restricted-globals +const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService()); +const config = loadConfig(loggerService); export const port = config.port; export const origin = config.url; export const host = new URL(config.url).host; diff --git a/packages/backend/tsconfig.backend.json b/packages/backend/tsconfig.backend.json new file mode 100644 index 0000000000..bab1ba2ad9 --- /dev/null +++ b/packages/backend/tsconfig.backend.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../shared/tsconfig.node.jsonc", + "compilerOptions": { + // Input + "rootDir": "./src", + + // Checking + "verbatimModuleSyntax": false, + "noImplicitOverride": false, + "noImplicitAny": false, + "strictFunctionTypes": false, + "strictPropertyInitialization": false, + + "paths": { + "@/*": ["./src/*"] + }, + + // Output + "removeComments": false, + "outDir": "./built" + }, + "include": [ + "./src/**/*.ts" + ], + "exclude": [ + "node_modules", + "**/built/", + "**/*.test.ts", + "./src/server/web/**/*.js", + "./src/server/web/**/*.mjs", + "./src/server/web/**/*.cjs", + "./src/server/web/**/*.d.ts", + "test/**/*", + "test-federation/**/*", + "test-server/**/*" + ] +} diff --git a/packages/backend/tsconfig.frontend.json b/packages/backend/tsconfig.frontend.json new file mode 100644 index 0000000000..cb7a8e1a1d --- /dev/null +++ b/packages/backend/tsconfig.frontend.json @@ -0,0 +1,27 @@ +{ + "extends": "../shared/tsconfig.web.jsonc", + "compilerOptions": { + "noEmit": true, + }, + "include": [ + "./assets/**/*.js", + "./assets/**/*.mjs", + "./assets/**/*.cjs", + "./assets/**/*.d.ts", + "./src/server/web/**/*.js", + "./src/server/web/**/*.mjs", + "./src/server/web/**/*.cjs", + "./src/server/web/**/*.d.ts" + ], + "exclude": [ + "node_modules", + "built", + "built-test", + "js-built", + "temp", + "coverage", + "test", + "test-federation", + "test-server" + ] +} diff --git a/packages/backend/tsconfig.json b/packages/backend/tsconfig.json index afed1f186c..eece3feb07 100644 --- a/packages/backend/tsconfig.json +++ b/packages/backend/tsconfig.json @@ -1,53 +1,10 @@ { - "compilerOptions": { - "allowJs": true, - "noEmitOnError": true, - "noImplicitAny": true, - "noImplicitReturns": true, - "noUnusedParameters": false, - "noUnusedLocals": false, - "noFallthroughCasesInSwitch": true, - "declaration": false, - "sourceMap": false, - "target": "ES2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "allowSyntheticDefaultImports": true, - "removeComments": false, - "noLib": false, - "strict": true, - "strictNullChecks": true, - "strictPropertyInitialization": false, - "skipLibCheck": true, - "experimentalDecorators": true, - "emitDecoratorMetadata": true, - "resolveJsonModule": true, - "isolatedModules": true, - "incremental": true, - "rootDir": "./src", - "baseUrl": "./", - "paths": { - "@/*": ["./src/*"] - }, - "outDir": "./built", - "types": [ - "node" - ], - "typeRoots": [ - "./src/@types", - "./node_modules/@types", - "./node_modules" - ], - "lib": [ - "esnext" - ] - }, - "compileOnSave": false, - "include": [ - "./src/**/*.ts" - ], - "exclude": [ - "node_modules", - "./src/**/*.test.ts" + "$schema": "https://json.schemastore.org/tsconfig", + "files": [], + // WebStorm only reads one tsconfig per directory, so this tricks it into loading both. + "references": [ + { "path": "./tsconfig.scripts.json" }, + { "path": "./tsconfig.backend.json" }, + { "path": "./tsconfig.frontend.json" } ] } diff --git a/packages/backend/tsconfig.scripts.json b/packages/backend/tsconfig.scripts.json new file mode 100644 index 0000000000..97dcf29e00 --- /dev/null +++ b/packages/backend/tsconfig.scripts.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../shared/tsconfig.scripts.jsonc", + "include": [ + "jest.*", + "eslint.*", + "scripts/**/*" + ], + "exclude": [ + "node_modules", + "built", + "built-test", + "js-built", + "temp", + "coverage", + "ormconfig.js", + "scripts/check_connect.js", + "scripts/generate_api_json.js" + ] +} diff --git a/packages/frontend-embed/.gitignore b/packages/frontend-embed/.gitignore index 1aa0ac14e8..c9047f496e 100644 --- a/packages/frontend-embed/.gitignore +++ b/packages/frontend-embed/.gitignore @@ -1 +1,2 @@ /storybook-static +tsconfig.json.bak diff --git a/packages/frontend-embed/eslint.config.js b/packages/frontend-embed/eslint.config.js index fbe91f3660..bf18d7c8df 100644 --- a/packages/frontend-embed/eslint.config.js +++ b/packages/frontend-embed/eslint.config.js @@ -15,7 +15,7 @@ export default [ }, ...pluginVue.configs['flat/recommended'], { - files: ['{src,test,js,@types}/**/*.{ts,vue}'], + files: ['{src,test,js,@types}/**/*.{ts,vue}', 'vue-shims.d.ts'], plugins: { sharkey: { rules: { locale: localeRule } } }, languageOptions: { globals: { @@ -41,7 +41,7 @@ export default [ parserOptions: { extraFileExtensions: ['.vue'], parser: tsParser, - project: ['./tsconfig.json'], + project: ['tsconfig.vue.json'], sourceType: 'module', tsconfigRootDir: import.meta.dirname, }, @@ -99,12 +99,47 @@ export default [ }, }, { + files: [ + '*.js', + '*.ts', + 'lib/**/*.ts', + 'lib/**/*.js', + 'scripts/**/*.ts', + 'scripts/**/*.js', + 'scripts/**/*.mjs', + 'scripts/**/*.cjs', + ], ignores: [ - "**/lib/", - "**/temp/", - "**/built/", - "**/coverage/", - "**/node_modules/", - ] + 'node_modules', + 'vue-shims.d.ts', + 'src', + 'test', + '@types', + 'assets', + ], + languageOptions: { + globals: { + ...globals.node, + }, + parserOptions: { + parser: tsParser, + project: ['tsconfig.scripts.json'], + sourceType: 'module', + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + 'import/no-default-export': 'off', + }, + }, + { + ignores: [ + '**/lib/', + '**/temp/', + '**/built/', + '**/coverage/', + '**/node_modules/', + 'vue-shims.d.ts', + ], }, ]; diff --git a/packages/frontend-embed/package.json b/packages/frontend-embed/package.json index 2a5491956d..5ead73b132 100644 --- a/packages/frontend-embed/package.json +++ b/packages/frontend-embed/package.json @@ -3,68 +3,71 @@ "private": true, "type": "module", "scripts": { - "watch": "vite", - "build": "vite build", - "typecheck": "vue-tsc --noEmit", - "eslint": "eslint --quiet \"{src,test,js,@types}/**/*.{js,jsx,ts,tsx,vue}\" --cache", + "watch": "node scripts/build.mjs --watch", + "build": "node scripts/build.mjs", + "build:pre": "pnpm run -w build-pre && pnpm run --filter misskey-js build && pnpm run --filter sw build && pnpm run --filter frontend-shared build", + "typecheck-all": "pnpm run --no-bail typecheck:vue && pnpm run --no-bail typecheck:scripts", + "typecheck": "pnpm run typecheck:vue && pnpm run typecheck:scripts", + "typecheck:vue": "vue-tsc -p tsconfig.vue.json --noEmit", + "typecheck:scripts": "tsc -p tsconfig.scripts.json --noEmit", + "eslint": "eslint --quiet --cache -c eslint.config.js .", "lint": "pnpm typecheck && pnpm eslint" }, "dependencies": { - "@discordapp/twemoji": "15.1.0", + "@discordapp/twemoji": "16.0.1", "@phosphor-icons/web": "2.1.2", - "mfm-js": "npm:@transfem-org/sfm-js@0.24.8", "buraha": "0.0.1", "frontend-shared": "workspace:*", "json5": "2.2.3", + "mfm-js": "npm:@transfem-org/sfm-js@0.26.1", "misskey-js": "workspace:*", "punycode.js": "2.3.1", - "shiki": "3.3.0", + "shiki": "3.13.0", "tinycolor2": "1.6.0", - "uuid": "11.1.0", - "vue": "3.5.14" + "uuid": "13.0.0", + "vue": "3.5.21" }, "devDependencies": { - "@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.2", + "@misskey-dev/eslint-plugin": "2.1.0", + "@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.3", "@rollup/plugin-json": "6.1.0", "@rollup/plugin-replace": "6.0.2", - "@rollup/pluginutils": "5.1.4", + "@rollup/pluginutils": "5.3.0", "@testing-library/vue": "8.1.0", - "@twemoji/parser": "15.1.1", - "@types/estree": "1.0.7", + "@types/estree": "1.0.8", "@types/micromatch": "4.0.9", - "@types/node": "22.15.2", + "@types/node": "22.18.1", "@types/punycode.js": "npm:@types/punycode@2.1.4", "@types/tinycolor2": "1.4.6", "@types/ws": "8.18.1", - "@typescript-eslint/eslint-plugin": "8.31.0", - "@typescript-eslint/parser": "8.31.0", - "@vitejs/plugin-vue": "5.2.3", - "@vitest/coverage-v8": "3.1.2", - "@vue/compiler-sfc": "3.5.14", - "@vue/runtime-core": "3.5.14", - "acorn": "8.14.1", + "@typescript-eslint/eslint-plugin": "8.44.1", + "@typescript-eslint/parser": "8.44.1", + "@vitejs/plugin-vue": "6.0.1", + "@vitest/coverage-v8": "3.2.4", + "@vue/runtime-core": "3.5.21", + "acorn": "8.15.0", "astring": "1.9.0", - "cross-env": "7.0.3", - "eslint-plugin-import": "2.31.0", - "eslint-plugin-vue": "10.0.0", + "cross-env": "10.0.0", + "eslint-plugin-import": "2.32.0", + "eslint-plugin-vue": "10.5.0", "estree-walker": "3.0.3", "fast-glob": "3.3.3", - "happy-dom": "17.4.4", + "happy-dom": "18.0.1", "intersection-observer": "0.12.2", "micromatch": "4.0.8", - "msw": "2.7.5", + "msw": "2.11.3", "nodemon": "3.1.10", - "prettier": "3.5.3", - "rollup": "4.40.0", - "sass": "1.87.0", - "start-server-and-test": "2.0.11", - "tsc-alias": "1.8.15", + "prettier": "3.6.2", + "rollup": "4.52.2", + "sass": "1.93.2", + "start-server-and-test": "2.1.2", + "tsc-alias": "1.8.16", "tsconfig-paths": "4.2.0", - "typescript": "5.8.3", - "vite": "6.3.4", + "typescript": "5.9.2", + "vite": "7.1.7", "vite-plugin-turbosnap": "1.0.3", - "vue-component-type-helpers": "2.2.10", - "vue-eslint-parser": "10.1.3", - "vue-tsc": "2.2.10" + "vue-component-type-helpers": "3.0.8", + "vue-eslint-parser": "10.2.0", + "vue-tsc": "3.0.8" } } diff --git a/packages/frontend-embed/scripts/build.mjs b/packages/frontend-embed/scripts/build.mjs new file mode 100644 index 0000000000..2f0883d3cc --- /dev/null +++ b/packages/frontend-embed/scripts/build.mjs @@ -0,0 +1,37 @@ +/** + * Hot-swaps tsconfig files to work around vite limitations. + * Based on idea from https://github.com/vitejs/vite/discussions/8483#discussioncomment-6830634 + */ + +import nodeFs from 'node:fs/promises'; +import nodePath from 'node:path'; +import { execa } from 'execa'; + +const rootDir = nodePath.resolve(import.meta.dirname, '../'); +const tsConfig = nodePath.resolve(rootDir, 'tsconfig.json'); +const tsConfigBak = nodePath.resolve(rootDir, 'tsconfig.json.bak'); +const tsConfigVue = nodePath.resolve(rootDir, 'tsconfig.vue.json'); + +const mode = process.argv.slice(2).includes('--watch') ? 'watch' : 'build'; + +console.log('Staging tsconfig.vue.json as tsconfig.json...'); +await nodeFs.rename(tsConfig, tsConfigBak); +await nodeFs.copyFile(tsConfigVue, tsConfig); + +try { + console.log('Starting vite...'); + await execa( + 'vite', + mode === 'build' + ? ['build'] + : [], + { + stdout: process.stdout, + stderr: process.stderr, + }, + ); +} finally { + console.log('Restoring original tsconfig.json...'); + await nodeFs.rm(tsConfig); + await nodeFs.rename(tsConfigBak, tsConfig); +} diff --git a/packages/frontend-embed/src/components/EmPagination.vue b/packages/frontend-embed/src/components/EmPagination.vue index 94a91305f4..998b4e0a21 100644 --- a/packages/frontend-embed/src/components/EmPagination.vue +++ b/packages/frontend-embed/src/components/EmPagination.vue @@ -195,7 +195,7 @@ async function init(): Promise { fetching.value = true; const params = props.pagination.params ? isRef(props.pagination.params) ? props.pagination.params.value : props.pagination.params : {}; await misskeyApi(props.pagination.endpoint, { - ...params, + ...params as object, limit: props.pagination.limit ?? 10, allowPartial: true, }).then(res => { @@ -231,7 +231,7 @@ const fetchMore = async (): Promise => { moreFetching.value = true; const params = props.pagination.params ? isRef(props.pagination.params) ? props.pagination.params.value : props.pagination.params : {}; await misskeyApi(props.pagination.endpoint, { - ...params, + ...params as object, limit: SECOND_FETCH_LIMIT, ...(props.pagination.offsetMode ? { offset: offset.value, @@ -295,7 +295,7 @@ const fetchMoreAhead = async (): Promise => { moreFetching.value = true; const params = props.pagination.params ? isRef(props.pagination.params) ? props.pagination.params.value : props.pagination.params : {}; await misskeyApi(props.pagination.endpoint, { - ...params, + ...params as object, limit: SECOND_FETCH_LIMIT, ...(props.pagination.offsetMode ? { offset: offset.value, diff --git a/packages/frontend-embed/src/workers/tsconfig.json b/packages/frontend-embed/src/workers/tsconfig.json index 39ba45ddbb..19d498d537 100644 --- a/packages/frontend-embed/src/workers/tsconfig.json +++ b/packages/frontend-embed/src/workers/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "lib": ["esnext", "webworker"], + "lib": ["ES2022", "WebWorker", "Webworker.Iterable"], "incremental": true } } diff --git a/packages/frontend-embed/tsconfig.json b/packages/frontend-embed/tsconfig.json index 8db5776c91..77e8ba1be6 100644 --- a/packages/frontend-embed/tsconfig.json +++ b/packages/frontend-embed/tsconfig.json @@ -1,58 +1,9 @@ { - "compilerOptions": { - "allowJs": true, - "noEmitOnError": false, - "noImplicitAny": false, - "noImplicitReturns": true, - "noUnusedParameters": false, - "noUnusedLocals": false, - "noFallthroughCasesInSwitch": true, - "declaration": false, - "sourceMap": false, - "target": "ES2022", - "module": "ES2022", - "moduleResolution": "Bundler", - "removeComments": false, - "noLib": false, - "strict": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "resolveJsonModule": true, - "allowSyntheticDefaultImports": true, - "isolatedModules": true, - "useDefineForClassFields": true, - "verbatimModuleSyntax": true, - "skipLibCheck": true, - "incremental": true, - "baseUrl": ".", - "paths": { - "@/*": ["./src/*"], - "@@/*": ["../frontend-shared/*"] - }, - "typeRoots": [ - "./@types", - "./node_modules/@types", - "./node_modules/@vue-macros", - "./node_modules" - ], - "types": [ - "vite/client" - ], - "lib": [ - "esnext", - "dom", - "dom.iterable" - ], - "jsx": "preserve" - }, - "compileOnSave": false, - "include": [ - "./src/**/*.ts", - "./src/**/*.vue", - "./@types/**/*.ts" - ], - "exclude": [ - "node_modules", - ".storybook/**/*" + "$schema": "https://json.schemastore.org/tsconfig", + "files": [], + // WebStorm only reads one tsconfig per directory, so this tricks it into loading both. + "references": [ + { "path": "./tsconfig.scripts.json" }, + { "path": "./tsconfig.vue.json" } ] } diff --git a/packages/frontend-embed/tsconfig.scripts.json b/packages/frontend-embed/tsconfig.scripts.json new file mode 100644 index 0000000000..77ab85f761 --- /dev/null +++ b/packages/frontend-embed/tsconfig.scripts.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../shared/tsconfig.scripts.jsonc", + "include": [ + "*.js", + "*.ts", + "scripts/**/*.ts", + "scripts/**/*.js", + "scripts/**/*.mjs", + "scripts/**/*.cjs" + ], + "exclude": [ + "node_modules", + "vue-shims.d.ts", + "src", + "test", + "@types", + "assets" + ] +} diff --git a/packages/frontend-embed/tsconfig.vue.json b/packages/frontend-embed/tsconfig.vue.json new file mode 100644 index 0000000000..36e05cb151 --- /dev/null +++ b/packages/frontend-embed/tsconfig.vue.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../shared/tsconfig.web.jsonc", + "compilerOptions": { + "noImplicitAny": false, + "noUnusedLocals": false, + "noUnusedParameters": false, + "experimentalDecorators": true, + "resolveJsonModule": true, + "allowSyntheticDefaultImports": true, + "paths": { + "@/*": ["./src/*"], + "@@/*": ["../frontend-shared/*"] + }, + "typeRoots": [ + "./@types", + "./node_modules/@types", + "./node_modules/@vue-macros", + "./node_modules" + ], + "types": [ + "vite/client" + ], + "jsx": "preserve" + }, + "include": [ + "./src/**/*.ts", + "./src/**/*.vue", + "./@types/**/*.ts", + "./vue-shims.d.ts" + ], + "exclude": [ + "node_modules", + ".storybook/**/*" + ] +} diff --git a/packages/frontend-embed/vite.config.ts b/packages/frontend-embed/vite.config.ts index 1cd47b2754..52c6d12cea 100644 --- a/packages/frontend-embed/vite.config.ts +++ b/packages/frontend-embed/vite.config.ts @@ -1,13 +1,12 @@ import path from 'path'; import pluginVue from '@vitejs/plugin-vue'; import { type UserConfig, defineConfig } from 'vite'; +import { pluginReplaceIcons } from 'frontend-shared/util/vite.replaceIcons.js'; import { localesVersion } from '../../locales/version.js'; - import locales from '../../locales/index.js'; -import meta from '../../package.json'; +import meta from '../../package.json' with { type: 'json' }; import packageInfo from './package.json' with { type: 'json' }; import pluginJson5 from './vite.json5.js'; -import { pluginReplaceIcons } from '../frontend/vite.replaceIcons.js'; const extensions = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.json', '.json5', '.svg', '.sass', '.scss', '.css', '.vue']; @@ -85,13 +84,13 @@ export function getConfig(): UserConfig { '@/': __dirname + '/src/', '@@/': __dirname + '/../frontend-shared/', '/client-assets/': __dirname + '/assets/', - '/static-assets/': __dirname + '/../backend/assets/' + '/static-assets/': __dirname + '/../backend/assets/', }, }, css: { modules: { - generateScopedName(name, filename, _css): string { + generateScopedName(name, filename): string { const id = (path.relative(__dirname, filename.split('?')[0]) + '-' + name).replace(/[\\\/\.\?&=]/g, '-').replace(/(src-|vue-)/g, ''); const shortId = id.replace(/^(components(-global)?|widgets|ui(-_common_)?)-/, ''); return shortId + '-' + toBase62(hash(id)).substring(0, 4); @@ -99,6 +98,7 @@ export function getConfig(): UserConfig { }, preprocessorOptions: { scss: { + // @ts-expect-error not sure why this happens api: 'modern-compiler', }, }, @@ -163,6 +163,6 @@ export function getConfig(): UserConfig { }; } -const config = defineConfig(({ command, mode }) => getConfig()); +const config = defineConfig(() => getConfig()); export default config; diff --git a/packages/frontend-embed/vite.json5.ts b/packages/frontend-embed/vite.json5.ts index 87b67c2142..54dd5e4d84 100644 --- a/packages/frontend-embed/vite.json5.ts +++ b/packages/frontend-embed/vite.json5.ts @@ -1,9 +1,9 @@ // Original: https://github.com/rollup/plugins/tree/8835dd2aed92f408d7dc72d7cc25a9728e16face/packages/json import JSON5 from 'json5'; -import { Plugin } from 'rollup'; import { createFilter, dataToEsm } from '@rollup/pluginutils'; -import { RollupJsonOptions } from '@rollup/plugin-json'; +import type { Plugin } from 'rollup'; +import type { RollupJsonOptions } from '@rollup/plugin-json'; // json5 extends SyntaxError with additional fields (without subclassing) // https://github.com/json5/json5/blob/de344f0619bda1465a6e25c76f1c0c3dda8108d9/lib/parse.js#L1111-L1112 @@ -19,7 +19,6 @@ export default function json5(options: RollupJsonOptions = {}): Plugin { return { name: 'json5', - // eslint-disable-next-line no-shadow transform(json, id) { if (id.slice(-6) !== '.json5' || !filter(id)) return null; diff --git a/packages/frontend-shared/build.js b/packages/frontend-shared/build.js index f3a94fe364..e9d7de144e 100644 --- a/packages/frontend-shared/build.js +++ b/packages/frontend-shared/build.js @@ -16,11 +16,12 @@ const entryPoints = globSync('./js/**/**.{ts,tsx}'); const options = { entryPoints, minify: process.env.NODE_ENV === 'production', - outdir: './js-built', + outdir: './js-built/js', target: 'es2022', platform: 'browser', format: 'esm', sourcemap: 'linked', + tsconfig: 'tsconfig.web.json', }; const args = process.argv.slice(2).map(arg => arg.toLowerCase()); @@ -63,7 +64,7 @@ function buildDts() { return execa( 'tsc', [ - '--project', 'tsconfig.json', + '--project', 'tsconfig.web.json', '--outDir', 'js-built', '--declaration', 'true', '--emitDeclarationOnly', 'true', diff --git a/packages/frontend-shared/eslint.config.js b/packages/frontend-shared/eslint.config.js index ff4d27443b..87f709c938 100644 --- a/packages/frontend-shared/eslint.config.js +++ b/packages/frontend-shared/eslint.config.js @@ -5,11 +5,10 @@ import pluginVue from 'eslint-plugin-vue'; import pluginMisskey from '@misskey-dev/eslint-plugin'; import sharedConfig from '../shared/eslint.config.js'; -// eslint-disable-next-line import/no-default-export export default [ ...sharedConfig, { - files: ['**/*.vue'], + files: ['js/**/*.vue'], ...pluginMisskey.configs.typescript, }, ...pluginVue.configs['flat/recommended'], @@ -17,7 +16,7 @@ export default [ files: [ '@types/**/*.ts', 'js/**/*.ts', - '**/*.vue', + 'js/**/*.vue', ], languageOptions: { globals: { @@ -43,7 +42,7 @@ export default [ parserOptions: { extraFileExtensions: ['.vue'], parser: tsParser, - project: ['./tsconfig.json'], + project: ['./tsconfig.web.json'], sourceType: 'module', tsconfigRootDir: import.meta.dirname, }, @@ -98,17 +97,45 @@ export default [ 'vue/attribute-hyphenation': ['error', 'never'], }, }, + { + files: ['*.js', '*.ts'], + languageOptions: { + parserOptions: { + parser: tsParser, + project: ['./tsconfig.scripts.json'], + sourceType: 'module', + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + 'import/no-default-export': 'off', + }, + }, + { + files: ['util/**/*.ts', 'util/**/*.js'], + languageOptions: { + parserOptions: { + parser: tsParser, + project: ['./tsconfig.util.json'], + sourceType: 'module', + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + 'import/no-default-export': 'off', + }, + }, { ignores: [ // TODO: Error while loading rule '@typescript-eslint/naming-convention': Cannot use 'in' operator to search for 'type' in undefined のため一時的に無効化 // See https://github.com/misskey-dev/misskey/pull/15311 'js/i18n.ts', 'js-built/', - "**/lib/", - "**/temp/", - "**/built/", - "**/coverage/", - "**/node_modules/", - ] + '**/lib/', + '**/temp/', + '**/built/', + '**/coverage/', + '**/node_modules/', + ], }, ]; diff --git a/packages/frontend-shared/package.json b/packages/frontend-shared/package.json index b4a5dd89f5..0cd4f82cc4 100644 --- a/packages/frontend-shared/package.json +++ b/packages/frontend-shared/package.json @@ -14,27 +14,40 @@ } }, "scripts": { - "build": "node ./build.js", + "build": "pnpm run build:js && pnpm run build:util", + "build:js": "node ./build.js", + "build:util": "tsc -p tsconfig.util.json", "watch": "nodemon -w package.json -e json --exec \"node ./build.js --watch\"", - "eslint": "eslint --quiet \"{src,test,js,@types}/**/*.{js,jsx,ts,tsx,vue}\" --cache", - "typecheck": "tsc --noEmit", + "eslint": "eslint --quiet --cache -c eslint.config.js .", + "typecheck-all": "pnpm run --no-bail typecheck:scripts && pnpm run --no-bail typecheck:util && pnpm run --no-bail typecheck:web", + "typecheck": "pnpm run typecheck:scripts && pnpm run typecheck:util && pnpm run typecheck:web", + "typecheck:scripts": "tsc -p tsconfig.scripts.json --noEmit", + "typecheck:util": "tsc -p tsconfig.util.json --noEmit", + "typecheck:web": "tsc -p tsconfig.web.json --noEmit", "lint": "pnpm typecheck && pnpm eslint" }, "devDependencies": { - "@types/node": "22.15.2", - "@typescript-eslint/eslint-plugin": "8.31.0", - "@typescript-eslint/parser": "8.31.0", - "esbuild": "0.25.3", - "eslint-plugin-vue": "10.0.0", + "@types/node": "22.18.1", + "@typescript-eslint/eslint-plugin": "8.44.1", + "@typescript-eslint/parser": "8.44.1", + "@rollup/plugin-replace": "6.0.2", + "@rollup/pluginutils": "5.3.0", + "esbuild": "0.25.10", + "eslint": "9.36.0", + "eslint-plugin-import": "2.32.0", + "eslint-plugin-vue": "10.5.0", + "execa": "9.6.0", "nodemon": "3.1.10", - "typescript": "5.8.3", - "vue-eslint-parser": "10.1.3" + "rollup": "4.52.2", + "typescript": "5.9.2", + "vue-eslint-parser": "10.2.0" }, "files": [ "js-built" ], "dependencies": { + "buraha": "0.0.1", "misskey-js": "workspace:*", - "vue": "3.5.13" + "vue": "3.5.21" } } diff --git a/packages/frontend-shared/tsconfig.json b/packages/frontend-shared/tsconfig.json index 0512b50caf..973ca7a51c 100644 --- a/packages/frontend-shared/tsconfig.json +++ b/packages/frontend-shared/tsconfig.json @@ -1,44 +1,10 @@ { "$schema": "https://json.schemastore.org/tsconfig", - "compilerOptions": { - "target": "ES2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "declaration": true, - "declarationMap": true, - "sourceMap": false, - "outDir": "./js-built/", - "removeComments": true, - "resolveJsonModule": true, - "strict": true, - "strictFunctionTypes": true, - "strictNullChecks": true, - "experimentalDecorators": true, - "noImplicitReturns": true, - "esModuleInterop": true, - "verbatimModuleSyntax": true, - "skipLibCheck": true, - "incremental": true, - "baseUrl": ".", - "paths": { - "@/*": ["./*"], - "@@/*": ["./*"] - }, - "typeRoots": [ - "./@types", - "./node_modules/@types" - ], - "lib": [ - "esnext", - "dom" - ] - }, - "include": [ - "@types/**/*.ts", - "js/**/*" - ], - "exclude": [ - "node_modules", - "test/**/*" + "files": [], + // WebStorm only reads one tsconfig per directory, so this tricks it into loading both. + "references": [ + { "path": "./tsconfig.scripts.json" }, + { "path": "./tsconfig.util.json" }, + { "path": "./tsconfig.web.json" } ] } diff --git a/packages/frontend-shared/tsconfig.scripts.json b/packages/frontend-shared/tsconfig.scripts.json new file mode 100644 index 0000000000..a479a70325 --- /dev/null +++ b/packages/frontend-shared/tsconfig.scripts.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../shared/tsconfig.scripts.jsonc", + "include": [ + "*.js", + "*.ts" + ] +} diff --git a/packages/frontend-shared/tsconfig.util.json b/packages/frontend-shared/tsconfig.util.json new file mode 100644 index 0000000000..ba395d348b --- /dev/null +++ b/packages/frontend-shared/tsconfig.util.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../shared/tsconfig.lib.jsonc", + "compilerOptions": { + "noImplicitAny": false, + "outDir": "js-built/util", + "rootDir": "util" + }, + "include": [ + "util/**/*.js", + "util/**/*.ts" + ], + "exclude": [ + "node_modules", + "js-built" + ] +} diff --git a/packages/frontend-shared/tsconfig.web.json b/packages/frontend-shared/tsconfig.web.json new file mode 100644 index 0000000000..45372b05d8 --- /dev/null +++ b/packages/frontend-shared/tsconfig.web.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../shared/tsconfig.web.jsonc", + "compilerOptions": { + "module": "nodenext", + "moduleResolution": "nodenext", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "inlineSources": true, + "removeComments": false, + "outDir": "./js-built/", + "rootDir": ".", + "strictFunctionTypes": true, + "paths": { + "@/*": ["./*"], + "@@/*": ["./*"] + }, + "typeRoots": [ + "./@types", + "./node_modules/@types" + ] + }, + "include": [ + "./@types/**/*.ts", + "./js/**/*" + ], + "exclude": [ + "node_modules", + "js-built" + ] +} diff --git a/packages/frontend-shared/util/vite.replaceIcons.ts b/packages/frontend-shared/util/vite.replaceIcons.ts new file mode 100644 index 0000000000..0be8876c57 --- /dev/null +++ b/packages/frontend-shared/util/vite.replaceIcons.ts @@ -0,0 +1,432 @@ +import pluginReplace from '@rollup/plugin-replace'; +import type { RollupReplaceOptions } from '@rollup/plugin-replace'; +import type { Plugin } from 'rollup'; + +// https://github.com/rollup/plugins/issues/1541#issuecomment-3114729017 +const fix = (f: { default: T }): T => f as unknown as T; + +function iconsReplace(opts: RollupReplaceOptions): Plugin { + return fix(pluginReplace)({ + ...opts, + preventAssignment: false, + // only replace these strings at the start of strings, and make + // sure they're followed by a word-boundary that's not a dash + delimiters: ['(?<=["\'`])', '\\b(?!-)'], + }); +} + +export function pluginReplaceIcons(): Plugin[] { + return [ + iconsReplace({ + values: { + 'ti ti-alert-triangle': 'ph-warning ph-bold ph-lg', + }, + exclude: [ + '**/components/MkAnnouncementDialog.*', + '**/pages/announcement.*', + ], + }), + iconsReplace({ + values: { + 'ti ti-alert-triangle': 'ph-warning-circle ph-bold ph-lg', + }, + include: [ + '**/components/MkAnnouncementDialog.*', + '**/pages/announcement.*', + ], + }), + iconsReplace({ + values: { + 'ti ti-apps': 'ph-squares-four ph-bold ph-lg', + }, + include: [ + '**/pages/**', + '**/components/MkAuthConfirm.*', + ], + }), + iconsReplace({ + values: { + 'ti ti-apps': 'ph-stack ph-bold ph-lg', + }, + include: [ + '**/ui/**', + ], + }), + iconsReplace({ + values: { + 'ti ti-clock-play': 'ph-clock ph-bold ph-lg', + }, + exclude: [ + '**/components/MkMedia*', + ], + }), + iconsReplace({ + values: { + 'ti ti-clock-play': 'ph ph-gauge ph-bold ph-lg', + }, + include: [ + '**/components/MkMedia*', + ], + }), + iconsReplace({ + values: { + 'ti ti-photo': 'ph-image-square ph-bold ph-lg', + }, + exclude: [ + '**/pages/admin-user.*', + ], + }), + iconsReplace({ + values: { + 'ti ti-photo': 'ph-image ph-bold ph-lg', + }, + include: [ + '**/pages/admin-user.*', + ], + }), + iconsReplace({ + values: { + 'ti ti-reload': 'ph-arrow-clockwise ph-bold ph-lg', + }, + exclude: [ + '**/pages/settings/emoji-picker.*', + '**/pages/flash/flash.*', + '**/components/MkPageWindow.*', + ], + }), + iconsReplace({ + values: { + 'ti ti-reload': 'ph-arrow-counter-clockwise ph-bold ph-lg', + }, + include: [ + '**/pages/settings/emoji-picker.*', + ], + }), + iconsReplace({ + values: { + 'ti ti-reload': 'ph-arrows-clockwise ph-bold ph-lg', + }, + include: [ + '**/pages/flash/flash.*', + '**/components/MkPageWindow.*', + ], + }), + iconsReplace({ + values: { + 'ti ti-repeat': 'ph-rocket-launch ph-bold ph-lg', + }, + exclude: [ + '**/components/MkMedia*', + '**/scripts/get-user-menu.*', + '**/pages/gallery/post.*', + ], + }), + iconsReplace({ + values: { + 'ti ti-repeat': 'ph ph-repeat ph-bold ph-lg', + }, + include: [ + '**/components/MkMedia*', + '**/scripts/get-user-menu.*', + '**/pages/gallery/post.*', + ], + }), + iconsReplace({ + values: { + 'icon ti ti-brand-youtube': 'icon ph-youtube-logo ph-bold ph-lg', + 'ti ti ti-folder-symlink': 'sk-icons sk-foldermove sk-icons-lg', + 'ti ti-123': 'ph-numpad ph-bold ph-lg', + 'ti ti-access-point': 'ph-broadcast ph-bold ph-lg', + 'ti ti-activity': 'ph-pulse ph-bold ph-lg', + 'ti ti-ad': 'ph-flag ph-bold ph-lg', + 'ti ti-adjustments': 'ph-faders ph-bold ph-lg', + 'ti ti-align-box-left-bottom': 'ph-arrow-down-left ph-bold ph-lg', + 'ti ti-align-box-left-top': 'ph-arrow-up-left ph-bold ph-lg', + 'ti ti-align-box-right-bottom': 'ph-arrow-down-right ph-bold ph-lg', + 'ti ti-align-box-right-top': 'ph-arrow-up-right ph-bold ph-lg', + 'ti ti-align-left': 'ph-text-align-left ph-bold ph-lg', + 'ti ti-antenna': 'ph-flying-saucer ph-bold ph-lg', + 'ti ti-api': 'ph-key ph-bold ph-lg', + 'ti ti-app-window': 'ph-app-window ph-bold ph-lg', + 'ti ti-apple': 'ph-orange-slice ph-bold ph-lg', + 'ti ti-archive': 'ph-archive ph-bold ph-lg', + 'ti ti-arrow-back-up': 'ph-arrow-u-up-left ph-bold ph-lg', + 'ti ti-arrow-bar-to-down': 'ph-arrow-line-down ph-bold ph-lg', + 'ti ti-arrow-big-right': 'ph-arrow-fat-right ph-bold ph-lg', + 'ti ti-arrow-down': 'ph-arrow-down ph-bold ph-lg', + 'ti ti-arrow-left': 'ph-arrow-left ph-bold ph-lg', + 'ti ti-arrow-narrow-up': 'ph-arrow-up ph-bold ph-lg', + 'ti ti-arrow-right': 'ph-arrow-right ph-bold ph-lg', + 'ti ti-arrow-up': 'ph-arrow-up ph-bold ph-lg', + 'ti ti-arrows-maximize': 'ph-arrows-out ph-bold ph-lg', + 'ti ti-arrows-minimize': 'ph-arrows-in ph-bold ph-lg', + 'ti ti-arrows-move': 'ph-arrows-out-cardinal ph-bold ph-lg', + 'ti ti-arrows-sort': 'ph-arrows-down-up ph-bold ph-lg', + 'ti ti-arrows-up': 'ph-arrow-line-up ph-bold ph-lg', + 'ti ti-asterisk': 'ph-asterisk ph-bold ph-lg', + 'ti ti-at': 'ph-at ph-bold ph-lg', + 'ti ti-backspace': 'ph-backspace ph-bold ph-lg', + 'ti ti-badge': 'ph-seal-check ph-bold ph-lg', + 'ti ti-badges': 'ph-seal-check ph-bold ph-lg', + 'ti ti-ban': 'ph-prohibit ph-bold ph-lg', + 'ti ti-bell': 'ph-bell ph-bold ph-lg', + 'ti ti-bell-off': 'ph-bell ph-bold ph-lg', + 'ti ti-bell-plus': 'ph-bell-ringing ph-bold ph-lg', + 'ti ti-bell-ringing-2': 'ph-bell-ringing ph-bold ph-lg', + 'ti ti-bolt': 'ph-lightning ph-bold ph-lg', + 'ti ti-bookmark': 'ph-bookmark ph-bold ph-lg', + 'ti ti-brand-x': 'ph-twitter-logo ph-bold ph-lg', + 'ti ti-bulb': 'ph-lightbulb ph-bold ph-lg', + 'ti ti-cake': 'ph-cake ph-bold ph-lg', + 'ti ti-calendar': 'ph-calendar ph-bold ph-lg', + 'ti ti-calendar-time': 'ph-calendar ph-bold ph-lg', + 'ti ti-calendar-event': 'ph-calendar-star ph-bold ph-lg', + 'ti ti-camera': 'ph-camera ph-bold ph-lg', + 'ti ti-carousel-horizontal': 'ph-split-horizontal ph-bold ph-lg', + 'ti ti-carousel-vertical': 'ph-split-vertical ph-bold ph-lg', + 'ti ti-chart-arrows': 'ph-chart-bar-horizontal ph-bold ph-lg', + 'ti ti-chart-line': 'ph-chart-line ph-bold ph-lg', + 'ti ti-check': 'ph-check ph-bold ph-lg', + 'ti ti-checkbox': 'ph-check ph-bold ph-lg', + 'ti ti-checklist': 'ph-list-checks ph-bold ph-lg', + 'ti ti-checkup-list': 'ph-list-checks ph-bold ph-lg', + 'ti ti-chevron-double-right': 'ph-caret-double-right ph-bold ph-lg', + 'ti ti-chevron-left': 'ph-caret-left ph-bold ph-lg', + 'ti ti-chevron-right': 'ph-caret-right ph-bold ph-lg', + 'ti ti-chevron-up': 'ph-caret-up ph-bold ph-lg', + 'ti ti-chevrons-left': 'ph-caret-dobule-left ph-bold ph-lg', + 'ti ti-chevrons-right': 'ph-caret-right ph-bold ph-lg', + 'ti ti-circle': 'ph-circle ph-bold ph-lg', + 'ti ti-circle-check': 'ph-seal-check ph-bold ph-lg', + 'ti ti-circle-filled': 'ph-circle-half ph-bold ph-lg', + 'ti ti-circle-minus': 'ph-minus-circle ph-bold ph-lg', + 'ti ti-circle-x': 'ph-x-circle ph-bold ph-lg', + 'ti ti-clock': 'ph-clock ph-bold ph-lg', + 'ti ti-clock-edit': 'ph-pencil-simple ph-bold ph-lg', + 'ti ti-cloud': 'ph-cloud ph-bold ph-lg', + 'ti ti-code': 'ph-code ph-bold ph-lg', + 'ti ti-columns': 'ph-text-columns ph-bold ph-lg', + 'ti ti-comet': 'ph-shooting-star ph-bold ph-lg', + 'ti ti-confetti': 'ph-confetti ph-bold ph-lg', + 'ti ti-cookie': 'ph-cookie ph-bold ph-lg', + 'ti ti-copy': 'ph-copy ph-bold ph-lg', + 'ti ti-corner-up-right': 'ph-arrow-bend-up-right ph-bold ph-lg', + 'ti ti-cpu': 'ph-cpu ph-bold ph-lg', + 'ti ti-crop': 'ph-crop ph-bold ph-lg', + 'ti ti-crown': 'ph-crown ph-bold ph-lg', + 'ti ti-dashboard': 'ph-gauge ph-bold ph-lg', + 'ti ti-database': 'ph-database ph-bold ph-lg', + 'ti ti-device-desktop': 'ph-desktop ph-bold ph-lg', + 'ti ti-device-floppy': 'ph-floppy-disk ph-bold ph-lg', + 'ti ti-device-gamepad': 'ph-game-controller ph-bold ph-lg', + 'ti ti-device-mobile': 'ph-device-mobile ph-bold ph-lg', + 'ti ti-device-tablet': 'ph-device-tablet ph-bold ph-lg', + 'ti ti-device-tv': 'ph-television ph-bold ph-lg', + 'ti ti-device-usb': 'ph-usb ph-bold ph-lg', + 'ti ti-devices': 'ph-devices ph-bold ph-lg', + 'ti ti-dice': 'ph ph-dice-five ph-bold ph-lg', + 'ti ti-dice-5': 'ph ph-dice-five ph-bold ph-lg', + 'ti ti-dots': 'ph-dots-three ph-bold ph-lg', + 'ti ti-download': 'ph-download ph-bold ph-lg', + 'ti-download': 'ph-download ph-bold ph-lg', // in custom-emoji-manager.remote.list + 'ti ti-edit': 'ph-pencil-simple-line ph-bold ph-lg', + 'ti ti-equal-double': 'ph-equals ph-bold ph-lg', + 'ti ti-equal-not': 'ph-prohibit ph-bold ph-lg', + 'ti ti-eraser': 'ph-eraser ph-bold ph-lg', + 'ti ti-exclamation-circle': 'ph-warning-circle ph-bold ph-lg', + 'ti ti-external-link': 'ph-arrow-square-out ph-bold ph-lg', + 'ti ti-eye': 'ph-eye ph-bold ph-lg', + 'ti ti-eye-exclamation': 'ph-eye-slash ph-bold ph-lg', + 'ti ti-eye-off': 'ph-eye-slash ph-bold ph-lg', + 'ti ti-feather': 'ph-feather ph-bold ph-lg', + 'ti ti-file': 'ph-file ph-bold ph-lg', + 'ti ti-file-invoice': 'ph-newspaper-clipping ph-bold ph-lg', + 'ti ti-file-music': 'ph-file-audio ph-bold ph-lg', + 'ti ti-file-text': 'ph-file-text ph-bold ph-lg', + 'ti ti-file-zip': 'ph-file-zip ph-bold ph-lg', + 'ti ti-filter': 'ph-funnel ph-bold ph-lg', + 'ti ti-fingerprint': 'ph-fingerprint ph-bold ph-lg', + 'ti ti-flare': 'ph-fire ph-bold ph-lg', + 'ti ti-flask': 'ph-flask ph-bold ph-lg', + 'ti ti-folder': 'ph-folder ph-bold ph-lg', + 'ti ti-folder-plus': 'ph-folder-plus ph-bold ph-lg', + 'ti ti-folder-symlink': 'sk-icons sk-foldermove sk-icons-lg', + 'ti ti-forms': 'ph-textbox ph-bold ph-lg', + 'ti ti-ghost': 'ph-ghost ph-bold ph-lg', + 'ti ti-grid-dots': 'ph-dots-nine ph-bold ph-lg', + 'ti ti-hash': 'ph-hash ph-bold ph-lg', + 'ti ti-heart': 'ph-heart ph-bold ph-lg', + 'ti ti-heart-filled': 'ph-heart ph-bold ph-lg', + 'ti ti-heart-off': 'ph-heart-break ph-bold ph-lg', + 'ti ti-heart-plus': 'ph-heart ph-bold ph-lg', + 'ti ti-help-circle': 'ph-question ph-bold ph-lg', + 'ti ti-home': 'ph-house ph-bold ph-lg', + 'ti ti-hourglass-empty': 'ph-hourglass ph-bold ph-lg', + 'ti ti-icons': 'ph-squares-four ph-bold ph-lg', + 'ti-icons': 'ph-squares-four ph-bold ph-lg', // in custom-emoji-manager.local.list + 'ti ti-id': 'ph-identification-card ph-bold ph-lg', + 'ti ti-info-circle': 'ph-info ph-bold ph-lg', + 'ti ti-json': 'ph-brackets-curly ph-bold ph-lg', + 'ti ti-key': 'ph-key ph-bold ph-lg', + 'ti ti-language-hiragana': 'ph-translate ph-bold ph-lg', + 'ti ti-leaf': 'ph-leaf ph-bold ph-lg', + 'ti ti-license': 'ph-notebook ph-bold ph-lg', + 'ti ti-link': 'ph-link ph-bold ph-lg', + 'ti ti-link-off': 'ph-link-break ph-bold ph-lg', + 'ti ti-list': 'ph-list ph-bold ph-lg', + 'ti ti-list-numbers': 'ph-list-numbers ph-bold ph-lg', + 'ti ti-list-search': 'ph-list ph-bold ph-lg', + 'ti ti-lock': 'ph-lock ph-bold ph-lg', + 'ti ti-lock-open': 'ph-lock-open ph-bold ph-lg', + 'ti ti-lock-star': 'ph-shield-star ph-bold ph-lg', + 'ti ti-login-2': 'ph-sign-in ph-bold ph-lg', + 'ti ti-mail': 'ph-envelope ph-bold ph-lg', + 'ti-mail': 'ph-envelope ph-bold ph-lg', // in notification-recipient.item.vue + 'ti ti-map-pin': 'ph-map-pin ph-bold ph-lg', + 'ti ti-maximize': 'ph-frame-corners ph-bold ph-lg', + 'ti ti-medal': 'ph-trophy ph-bold ph-lg', + 'ti ti-menu': 'ph-list ph-bold ph-lg', + 'ti ti-menu-2': 'ph-list ph-bold ph-lg', + 'ti ti-message': 'ph-envelope ph-bold ph-lg', + 'ti ti-message-2': 'ph-envelope ph-bold ph-lg', + 'ti ti-message-exclamation': 'ph-exclamation ph-bold ph-lg', + 'ti ti-message-off': 'ph-bell-slash ph-bold ph-lg', + 'ti ti-message-x': 'ph-prohibit ph-bold ph-lg', + 'ti ti-messages': 'ph-envelope ph-bold ph-lg', + 'ti ti-messages-off': 'ph-envelope-open ph-bold ph-lg', + 'ti ti-minimize': 'ph-arrows-in-simple ph-bold ph-lg', + 'ti ti-minus': 'ph-minus ph-bold ph-lg', + 'ti ti-mood-happy': 'ph-smiley ph-bold ph-lg', + 'ti ti-mood-smile': 'ph-smiley ph-bold pg-lg', + 'ti ti-moon': 'ph-moon ph-bold ph-lg', + 'ti ti-movie': 'ph-film-strip ph-bold ph-lg', + 'ti ti-music': 'ph-music-notes ph-bold ph-lg', + 'ti ti-news': 'ph-newspaper ph-bold ph-lg', + 'ti ti-note': 'ph-note ph-bold ph-lg', + 'ti ti-notes': 'ph-notepad ph-bold ph-lg', + 'ti ti-notebook': 'ph-notebook ph-bold ph-lg', + 'ti ti-package': 'ph-package ph-bold ph-lg', + 'ti ti-paint': 'ph-paint-roller ph-bold ph-lg', + 'ti ti-palette': 'ph-palette ph-bold ph-lg', + 'ti ti-paperclip': 'ph-paperclip ph-bold ph-lg', + 'ti ti-password': 'ph-password ph-bold ph-lg', + 'ti ti-pencil': 'ph-pencil-simple ph-bold ph-lg', + 'ti ti-pencil-plus': 'ph-plus ph-bold pg-lg', + 'ti ti-photo-plus': 'ph-image-square ph-bold ph-lg', + 'ti ti-picture-in-picture': 'ph-picture-in-picture ph-bold ph-lg', + 'ti ti-pin': 'ph-push-pin ph-bold ph-lg', + 'ti ti-pinned-off': 'ph-push-pin-slash ph-bold ph-lg', + 'ti ti-plane': 'ph-airplane ph-bold ph-lg', + 'ti ti-plane-arrival': 'ph-airplane-landing ph-bold ph-lg', + 'ti ti-plane-departure': 'ph-airplane-takeoff ph-bold ph-lg', + 'ti ti-planet': 'ph-planet ph-bold ph-lg', + 'ti ti-planet-off': 'ph-globe-simple ph-bold ph-lg', + 'ti ti-player-eject': 'ph-eject ph-bold ph-lg', + 'ti ti-player-pause': 'ph-pause ph-bold ph-lg', + 'ti ti-player-pause-filled': 'ph-pause ph-bold ph-lg', + 'ti ti-player-play': 'ph-play ph-bold ph-lg', + 'ti ti-player-play-filled': 'ph-play ph-bold ph-lg', + 'ti ti-player-stop': 'ph-stop ph-bold ph-lg', + 'ti ti-player-track-next': 'ph-skip-forward ph-bold ph-lg', + 'ti ti-plug': 'ph-plug ph-bold ph-lg', + 'ti ti-plus': 'ph-plus ph-bold ph-lg', + 'ti ti-point': 'ph-circle ph-bold ph-lg', + 'ti ti-power': 'ph-power ph-bold ph-lg', + 'ti ti-presentation': 'ph-presentation ph-bold ph-lg', + 'ti ti-quote': 'ph-quotes ph-bold ph-lg', + 'ti ti-rectangle': 'ph-frame-corners ph-bold ph-lg', + 'ti ti-refresh': 'ph-arrows-counter-clockwise ph-bold ph-lg', + 'ti ti-repeat-off': 'ph-repeat ph-bold ph-lg', + 'ti ti-restore': 'ph-box-arrow-up ph-box ph-lg', + 'ti ti-robot': 'ph-robot ph-bold ph-lg', + 'ti ti-rocket': 'ph-rocket-launch ph-bold ph-lg', + 'ti ti-rocket-off': 'ph-rocket ph-bold ph-lg', + 'ti ti-rss': 'ph-rss ph-bold ph-lg', + 'ti ti-search': 'ph-magnifying-glass ph-bold ph-lg', + 'ti ti-section': 'ph-selection-all ph-bold ph-lg', + 'ti ti-selector': 'ph-caret-up-down ph-bold ph-lg', + 'ti ti-send': 'ph-paper-plane-tilt ph-bold ph-lg', + 'ti ti-server': 'ph-hard-drives ph-bold ph-lg', + 'ti ti-settings': 'ph-gear ph-bold ph-lg', + 'ti ti-share': 'ph-share-network ph-bold ph-lg', + 'ti ti-shield': 'ph-shield ph-bold ph-lg', + 'ti ti-shield-lock': 'ph-shield ph-bold ph-lg', + 'ti ti-slash': 'ph-check-fat ph-bold ph-lg', + 'ti ti-snowflake': 'ph-snowflake ph-bold ph-lg', + 'ti ti-sort-ascending-letters': 'ph-sort-ascending ph-bold ph-lg', + 'ti ti-sort-descending-letters': 'ph-sort-descending ph-bold ph-lg', + 'ti ti-sparkles': 'ph-sparkle ph-bold ph-lg', + 'ti ti-speakerphone': 'ph-megaphone ph-bold ph-lg', + 'ti ti-stack-2': 'ph-stack ph-bold ph-lg', + 'ti ti-star': 'ph-star ph-bold ph-lg', + 'ti ti-star-off': 'ph-star-half ph-bold ph-lg', + 'ti ti-sun': 'ph-sun ph-bold ph-lg', + 'ti ti-switch-horizontal': 'ph-arrows-left-right ph-bold ph-lg', + 'ti ti-terminal-2': 'ph-terminal-window ph-bold ph-lg', + 'ti ti-text-caption': 'ph-text-indent ph-bold ph-lg', + 'ti ti-tool': 'ph-wrench ph-bold ph-lg', + 'ti ti-trash': 'ph-trash ph-bold ph-lg', + 'ti-trash': 'ph-trash ph-bold ph-lg', // in custom-emoji-manager.local.list + 'ti ti-trophy': 'ph-trophy ph-bold ph-lg', + 'ti ti-universe': 'ph-rocket-launch ph-bold ph-lg', + 'ti ti-upload': 'ph-upload ph-bold ph-lg', + 'ti ti-user': 'ph-user ph-bold ph-lg', + 'ti ti-user-check': 'ph-user-check ph-bold ph-lg', + 'ti ti-user-circle': 'ph-user-circle ph-bold ph-lg', + 'ti ti-user-edit': 'ph-user-list ph-bold ph-lg', + 'ti ti-user-exclamation': 'ph-warning-circle ph-bold ph-lg', + 'ti ti-user-off': 'ph-user-minus ph-bold ph-lg', + 'ti ti-user-plus': 'ph-user-plus ph-bold ph-lg', + 'ti ti-user-question': 'ph-user-circle-dashed ph-bold ph-lg', + 'ti ti-user-search': 'ph-user-circle ph-bold ph-lg', + 'ti ti-user-shield': 'ph-newspaper-clipping ph-bold ph-lg', + 'ti ti-user-star': 'ph-user-focus ph-bold ph-lg', + 'ti ti-user-x': 'ph-prohibit ph-bold ph-lg', + 'ti ti-users': 'ph-users ph-bold ph-lg', + 'ti ti-video': 'ph-video ph-bold ph-lg', + 'ti ti-volume': 'ph-speaker-high ph-bold ph-lg', + 'ti ti-volume-3': 'ph-speaker-x ph-bold ph-lg', + 'ti ti-webhook': 'ph-webhooks-logo ph-bold ph-lg', + 'ti-webhook': 'ph-webhooks-logo ph-bold ph-lg', // in notification-recipient.item.vue + 'ti ti-whirl': 'ph-globe-hemisphere-west ph-bold ph-lg', + 'ti ti-window-maximize': 'ph-frame-corners ph-bold ph-lg', + 'ti ti-world': 'ph-globe-hemisphere-west ph-bold ph-lg', + 'ti ti-world-download': 'ph-cloud-arrow-down ph-bold ph-lg', + 'ti ti-world-cog': 'ph-gear-six ph-bold ph-lg', + 'ti ti-world-search': 'ph-binoculars ph-bold ph-lg', + 'ti ti-world-upload': 'ph-cloud-arrow-up ph-bold ph-lg', + 'ti ti-world-x': 'ph-planet ph-bold ph-lg', + 'ti ti-x': 'ph-x ph-bold ph-lg', + 'ti ti-help': 'ph-question ph-bold ph-lg', + 'ti-help': 'ph-question ph-bold ph-lg', // in notification-recipient.item.vue + 'ti ti ti-caret-down': 'ph-caret-down ph-bold ph-lg', + 'ti ti-chevron-down': 'ph-caret-down ph-bold ph-lg', + 'ti ti-accessible': 'ph-person-simple-circle ph-bold ph-lg', + 'ti ti-antenna-bars-3': 'ph-cell-signal-medium ph-bold ph-lg', + 'ti ti-arrows-horizontal': 'ph-arrows-horizontal ph-bold ph-lg', + 'ti ti-battery-vertical-eco': 'ph-battery-plus-vertical ph-bold ph-lg', + 'ti ti-caret-down': 'ph-caret-down ph-bold ph-lg', + 'ti ti-clipboard': 'ph-clipboard ph-bold ph-lg', + 'ti ti-cloud-cog': 'ph-cloud-check ph-bold ph-lg', + 'ti ti-cloud-down': 'ph-cloud-arrow-down ph-bold ph-lg', + 'ti ti-cloud-up': 'ph-cloud-arrow-up ph-bold ph-lg', + 'ti ti-dots-circle-horizontal': 'ph-dots-three-circle ph-bold ph-lg', + 'ti ti-mood-plus': 'ph-smiley ph-bold ph-lg', + 'ti ti-photo-exclamation': 'ph-gear ph-bold ph-lg', + 'ti ti-photo-search': 'ph-file-magnifying-glass ph-bold ph-lg', + 'ti ti-settings-2': 'ph-gear-six ph-bold ph-lg', + 'ti ti-settings-cog': 'ph-gear ph-bold ph-lg', + 'ti ti-ticket': 'ph-ticket ph-bold ph-lg', + 'ti ti-user-cog': 'ph-user-circle-gear ph-bold ph-lg', + 'ti ti-users-group': 'ph-users-three ph-bold ph-lg', + 'ti ti-code-asterisk': 'ph-brackets-curly ph-bold ph-lg', + 'ti ti-hourglass-high': 'ph-hourglass-high ph-bold ph-lg', + 'ti ti-http-que': 'ph-queue ph-bold ph-lg', + 'ti ti-list-check': 'ph-list-checks ph-bold ph-lg', + 'ti ti-logs': 'ph-list-dashes ph-bold ph-lg', + 'ti ti-timeline-event': 'ph-map-pin-simple-line ph-bold ph-lg', + 'ti ti-user-minus': 'ph-user-minus ph-bold ph-lg', + }, + }), + ]; +} diff --git a/packages/frontend/.gitignore b/packages/frontend/.gitignore index 1aa0ac14e8..c9047f496e 100644 --- a/packages/frontend/.gitignore +++ b/packages/frontend/.gitignore @@ -1 +1,2 @@ /storybook-static +tsconfig.json.bak diff --git a/packages/frontend/.storybook/charts.ts b/packages/frontend/.storybook/charts.ts index 31bb9e51c5..acc75c69ab 100644 --- a/packages/frontend/.storybook/charts.ts +++ b/packages/frontend/.storybook/charts.ts @@ -3,10 +3,10 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { HttpResponse, http } from 'msw'; +import { HttpResponse } from 'msw'; import type { DefaultBodyType, HttpResponseResolver, JsonBodyType, PathParams } from 'msw'; import seedrandom from 'seedrandom'; -import { action } from '@storybook/addon-actions'; +import { action } from 'storybook/actions'; function getChartArray(seed: string, limit: number, option?: { accumulate?: boolean, mul?: number }): number[] { const rng = seedrandom(seed); @@ -30,7 +30,9 @@ export function getChartResolver(fields: string[], option?: { accumulate?: boole action(`GET ${request.url}`)(); const limitParam = new URL(request.url).searchParams.get('limit'); const limit = limitParam ? parseInt(limitParam) : 30; - const res = {}; + // What the *fuck* is the type of this object??? + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const res: any = {}; for (const field of fields) { const layers = field.split('.'); let current = res; diff --git a/packages/frontend/.storybook/fake-utils.ts b/packages/frontend/.storybook/fake-utils.ts index 44e2263ca0..29ceb311f9 100644 --- a/packages/frontend/.storybook/fake-utils.ts +++ b/packages/frontend/.storybook/fake-utils.ts @@ -12,7 +12,7 @@ export const firstNameDict = [ 'Ethan', 'Olivia', 'Jackson', 'Emma', 'Liam', 'Ava', 'Aiden', 'Sophia', 'Mason', 'Isabella', 'Noah', 'Mia', 'Lucas', 'Harper', 'Caleb', 'Abigail', 'Samuel', 'Emily', 'Logan', 'Madison', 'Benjamin', 'Chloe', 'Elijah', 'Grace', 'Alexander', 'Scarlett', 'William', 'Zoey', 'James', 'Lily', -] +]; /** * AIで生成した無作為なラストネーム @@ -21,7 +21,7 @@ export const lastNameDict = [ 'Anderson', 'Johnson', 'Thompson', 'Davis', 'Rodriguez', 'Smith', 'Patel', 'Williams', 'Lee', 'Brown', 'Garcia', 'Jackson', 'Martinez', 'Taylor', 'Harris', 'Nguyen', 'Miller', 'Jones', 'Wilson', 'White', 'Thomas', 'Garcia', 'Martinez', 'Robinson', 'Turner', 'Lewis', 'Hall', 'King', 'Baker', 'Cooper', -] +]; /** * AIで生成した無作為な国名 @@ -30,7 +30,7 @@ export const countryDict = [ 'Japan', 'Canada', 'Brazil', 'Australia', 'Italy', 'SouthAfrica', 'Mexico', 'Sweden', 'Russia', 'India', 'Germany', 'Argentina', 'South Korea', 'France', 'Nigeria', 'Turkey', 'Spain', 'Egypt', 'Thailand', 'Vietnam', 'Kenya', 'Saudi Arabia', 'Netherlands', 'Colombia', 'Poland', 'Chile', 'Malaysia', 'Ukraine', 'New Zealand', 'Peru', -] +]; export function text(length: number = 10, seed?: string): string { let result = ""; @@ -140,7 +140,7 @@ export function imageDataUrl(options?: { throw new Error('Failed to get 2d context'); } - ctx.beginPath() + ctx.beginPath(); const red = options?.color?.red ?? integer(0, 255, seed); const green = options?.color?.green ?? integer(0, 255, seed); diff --git a/packages/frontend/.storybook/fakes.ts b/packages/frontend/.storybook/fakes.ts index 7288dabb60..b9e7008c24 100644 --- a/packages/frontend/.storybook/fakes.ts +++ b/packages/frontend/.storybook/fakes.ts @@ -4,7 +4,7 @@ */ import { AISCRIPT_VERSION } from '@syuilo/aiscript'; -import type { entities } from 'misskey-js' +import type { entities } from 'misskey-js'; import { date, imageDataUrl, text } from "./fake-utils.js"; export function abuseUserReport() { @@ -124,7 +124,7 @@ export function galleryPost(isSensitive = false) { isSensitive, likedCount: 0, isLiked: false, - } + }; } export function file(isSensitive = false) { @@ -220,6 +220,11 @@ export function federationInstance(): entities.FederationInstance { themeColor: '', infoUpdatedAt: '', latestRequestReceivedAt: '', + isMediaSilenced: false, + rejectReports: false, + rejectQuotes: false, + isBubbled: false, + mandatoryCW: null, }; } @@ -240,6 +245,13 @@ export function note(id = 'somenoteid'): entities.Note { reactionCount: 0, renoteCount: 0, repliesCount: 0, + threadId: '', + userHost: null, + isMutingThread: false, + isMutingNote: false, + isFavorited: false, + isRenoted: false, + bypassSilence: false, }; } @@ -254,6 +266,23 @@ export function userLite(id = 'someuserid', username = 'miskist', host: entities avatarBlurhash: 'eQFRshof5NWBRi},juayfPju53WB?0ofs;s*a{ofjuay^SoMEJR%ay', avatarDecorations: [], emojis: {}, + createdAt: '', + updatedAt: null, + lastFetchedAt: null, + approved: false, + description: null, + isAdmin: false, + isModerator: false, + isSystem: false, + noindex: false, + enableRss: false, + mandatoryCW: null, + isSilenced: false, + bypassSilence: false, + followersCount: 0, + followingCount: 0, + notesCount: 0, + attributionDomains: [], }; } @@ -312,19 +341,27 @@ export function userDetailed(id = 'someuserid', username = 'miskist', host: enti alsoKnownAs: null, notify: 'none', memo: null, + backgroundUrl: null, + backgroundId: null, + backgroundBlurhash: null, + listenbrainz: null, + canChat: true, + chatScope: 'none', }; } +export default userDetailed; + export function inviteCode(isUsed = false, hasExpiration = false, isExpired = false, isCreatedBySystem = false) { const date = new Date(); const createdAt = new Date(); - createdAt.setDate(date.getDate() - 1) + createdAt.setDate(date.getDate() - 1); const expiresAt = new Date(); if (isExpired) { - expiresAt.setHours(date.getHours() - 1) + expiresAt.setHours(date.getHours() - 1); } else { - expiresAt.setHours(date.getHours() + 1) + expiresAt.setHours(date.getHours() + 1); } return { @@ -336,7 +373,7 @@ export function inviteCode(isUsed = false, hasExpiration = false, isExpired = fa usedBy: isUsed ? userDetailed('3i3r2znx1v') : null, usedAt: isUsed ? date.toISOString() : null, used: isUsed, - } + }; } export function role(params: { @@ -382,10 +419,11 @@ export function role(params: { condFormula: { id: '', type: 'or', - values: [] + values: [], }, policies: {}, - } + preserveAssignmentOnMoveAccount: true, + }; } export function emoji(params?: { @@ -401,7 +439,7 @@ export function emoji(params?: { license?: string, isSensitive?: boolean, localOnly?: boolean, - roleIdsThatCanBeUsedThisEmojiAsReaction?: {id:string, name:string}[], + roleIdsThatCanBeUsedThisEmojiAsReaction?: { id: string, name: string }[], updatedAt?: string, }, seed?: string): entities.EmojiDetailedAdmin { const _seed = seed ?? (params?.id ?? "DEFAULT_SEED"); @@ -409,7 +447,7 @@ export function emoji(params?: { const name = params?.name ?? text(8, _seed); const updatedAt = params?.updatedAt ?? date({}, _seed).toISOString(); - const image = imageDataUrl({}, _seed) + const image = imageDataUrl({}, _seed); return { id: id, @@ -426,5 +464,5 @@ export function emoji(params?: { localOnly: params?.localOnly ?? false, roleIdsThatCanBeUsedThisEmojiAsReaction: params?.roleIdsThatCanBeUsedThisEmojiAsReaction ?? [], updatedAt: updatedAt, - } + }; } diff --git a/packages/frontend/.storybook/generate.tsx b/packages/frontend/.storybook/generate.tsx index 89d4214141..36b6bd1340 100644 --- a/packages/frontend/.storybook/generate.tsx +++ b/packages/frontend/.storybook/generate.tsx @@ -23,7 +23,7 @@ interface ImportDeclaration extends estree.ImportDeclaration { const generator = { ...GENERATOR, - ImportDeclaration(node: ImportDeclaration, state: State) { + ImportDeclaration(node: ImportDeclaration, state: State): void { state.write('import '); if (node.kind === 'type') state.write('type '); const { specifiers } = node; @@ -63,7 +63,7 @@ const generator = { state.write(';'); }, - SatisfiesExpression(node: SatisfiesExpression, state: State) { + SatisfiesExpression(node: SatisfiesExpression, state: State): void { switch (node.expression.type) { case 'ArrowFunctionExpression': { state.write('('); @@ -72,7 +72,7 @@ const generator = { break; } default: { - // @ts-ignore + // @ts-expect-error Produces "Expression produces a union type that is too complex to represent" for some reason this[node.expression.type](node.expression, state); break; } @@ -94,7 +94,6 @@ type SplitCamel< : SplitCamel : YN; -// @ts-ignore type SplitKebab = T extends `${infer XH}-${infer XR}` ? [XH, ...SplitKebab] : [T]; @@ -110,7 +109,6 @@ type ToKebab = T extends readonly [ ? `${XH}${XR extends readonly string[] ? `-${ToKebab}` : ''}` : ''; -// @ts-ignore type ToPascal = T extends readonly [ infer XH extends string, ...infer XR extends readonly string[] @@ -126,6 +124,7 @@ function h( return Object.assign(props || {}, { type }) as T; } +// eslint-disable-next-line @typescript-eslint/no-namespace declare namespace h.JSX { type Element = estree.Node; type IntrinsicElements = { @@ -474,5 +473,5 @@ function toStories(component: string): Promise { await Promise.all(components.map(async (component) => { const stories = component.replace(/\.vue$/, '.stories.ts'); await writeFile(stories, await toStories(component)); - })) + })); })(); diff --git a/packages/frontend/.storybook/locale.d.ts b/packages/frontend/.storybook/locale.d.ts new file mode 100644 index 0000000000..030b930342 --- /dev/null +++ b/packages/frontend/.storybook/locale.d.ts @@ -0,0 +1,3 @@ +import locales from '../../../locales/index.js'; + +export default locales['ja-JP'] as const; diff --git a/packages/frontend/.storybook/main.ts b/packages/frontend/.storybook/main.ts index c1119c2523..88920fde1d 100644 --- a/packages/frontend/.storybook/main.ts +++ b/packages/frontend/.storybook/main.ts @@ -7,7 +7,7 @@ import { createRequire } from 'node:module'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import type { StorybookConfig } from '@storybook/vue3-vite'; -import { type Plugin, mergeConfig } from 'vite'; +import { mergeConfig } from 'vite'; import turbosnap from 'vite-plugin-turbosnap'; const require = createRequire(import.meta.url); @@ -29,19 +29,20 @@ const config = { options: {}, }, docs: { + // @ts-expect-error This seems to be wrong, but I can't find what the alternative might be. autodocs: 'tag', }, core: { disableTelemetry: true, }, async viteFinal(config) { - const replacePluginForIsChromatic = config.plugins?.findIndex((plugin: Plugin) => plugin && plugin.name === 'replace') ?? -1; + const replacePluginForIsChromatic = config.plugins?.findIndex(plugin => plugin && 'name' in plugin && plugin.name === 'replace') ?? -1; if (~replacePluginForIsChromatic) { config.plugins?.splice(replacePluginForIsChromatic, 1); } //pluginsからcreateSearchIndexを削除、複数あるかもしれないので全て削除 - config.plugins = config.plugins?.filter((plugin: Plugin) => plugin && plugin.name !== 'createSearchIndex') ?? []; + config.plugins = config.plugins?.filter(plugin => plugin && 'name' in plugin && plugin.name !== 'createSearchIndex') ?? []; return mergeConfig(config, { plugins: [ diff --git a/packages/frontend/.storybook/mocks.ts b/packages/frontend/.storybook/mocks.ts index 29cb112ccb..cc2e03cdb5 100644 --- a/packages/frontend/.storybook/mocks.ts +++ b/packages/frontend/.storybook/mocks.ts @@ -3,14 +3,15 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { type SharedOptions, http, HttpResponse } from 'msw'; +import { http, HttpResponse } from 'msw'; +import type { SharedOptions } from 'msw'; export const onUnhandledRequest = ((req, print) => { const url = new URL(req.url); if (url.hostname !== 'localhost' || /^\/(?:client-assets\/|fluent-emojis?\/|iframe.html$|node_modules\/|src\/|sb-|static-assets\/|vite\/)/.test(url.pathname)) { - return + return; } - print.warning() + print.warning(); }) satisfies SharedOptions['onUnhandledRequest']; export const commonHandlers = [ diff --git a/packages/frontend/.storybook/package.json b/packages/frontend/.storybook/package.json deleted file mode 100644 index bedb411a91..0000000000 --- a/packages/frontend/.storybook/package.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "type": "module" -} diff --git a/packages/frontend/.storybook/preload-locale.ts b/packages/frontend/.storybook/preload-locale.ts index c823ff9bee..6c9af0cafb 100644 --- a/packages/frontend/.storybook/preload-locale.ts +++ b/packages/frontend/.storybook/preload-locale.ts @@ -7,7 +7,7 @@ import { writeFile } from 'node:fs/promises'; import locales from '../../../locales/index.js'; await writeFile( - new URL('locale.ts', import.meta.url), - `export default ${JSON.stringify(locales['ja-JP'], undefined, 2)} as const;`, + new URL('locale.js', import.meta.url), + `export default ${JSON.stringify(locales['ja-JP'], undefined, 2)};`, 'utf8', -) +); diff --git a/packages/frontend/.storybook/preload-theme.ts b/packages/frontend/.storybook/preload-theme.ts index 1b6a605a6e..6d418495a9 100644 --- a/packages/frontend/.storybook/preload-theme.ts +++ b/packages/frontend/.storybook/preload-theme.ts @@ -30,16 +30,16 @@ const keys = [ 'd-u0', 'rosepine', 'rosepine-dawn', -] +]; await Promise.all(keys.map((key) => readFile(new URL(`../../frontend-shared/themes/${key}.json5`, import.meta.url), 'utf8'))).then((sources) => { writeFile( - new URL('./themes.ts', import.meta.url), + new URL('./themes.js', import.meta.url), `export default ${JSON.stringify( Object.fromEntries(sources.map((source, i) => [keys[i], JSON5.parse(source)])), undefined, 2, - )} as const;`, - 'utf8' + )};`, + 'utf8', ); }); diff --git a/packages/frontend/.storybook/preview.ts b/packages/frontend/.storybook/preview.ts index fb855c1410..ef04b9d68d 100644 --- a/packages/frontend/.storybook/preview.ts +++ b/packages/frontend/.storybook/preview.ts @@ -5,13 +5,15 @@ import { FORCE_RE_RENDER, FORCE_REMOUNT } from '@storybook/core-events'; import { addons } from '@storybook/preview-api'; -import { type Preview, setup } from '@storybook/vue3'; +import { setup } from '@storybook/vue3'; import isChromatic from 'chromatic/isChromatic'; import { initialize, mswLoader } from 'msw-storybook-addon'; import { userDetailed } from './fakes.js'; import locale from './locale.js'; import { commonHandlers, onUnhandledRequest } from './mocks.js'; import themes from './themes.js'; +import type { Preview } from '@storybook/vue3'; +import type * as MisskeyOS from '../src/os.js'; import '../src/style.scss'; const appInitialized = Symbol(); @@ -19,13 +21,13 @@ const appInitialized = Symbol(); let lastStory: string | null = null; let moduleInitialized = false; let unobserve = () => {}; -let misskeyOS = null; +let misskeyOS: typeof MisskeyOS | null = null; function loadTheme(applyTheme: typeof import('../src/theme')['applyTheme']) { unobserve(); - const theme = themes[window.document.documentElement.dataset.misskeyTheme]; + const theme = themes[window.document.documentElement.dataset.misskeyTheme as string]; if (theme) { - applyTheme(themes[window.document.documentElement.dataset.misskeyTheme]); + applyTheme(themes[window.document.documentElement.dataset.misskeyTheme as string]); } else { applyTheme(themes['l-light']); } @@ -33,9 +35,9 @@ function loadTheme(applyTheme: typeof import('../src/theme')['applyTheme']) { for (const entry of entries) { if (entry.attributeName === 'data-misskey-theme') { const target = entry.target as HTMLElement; - const theme = themes[target.dataset.misskeyTheme]; + const theme = themes[target.dataset.misskeyTheme as string]; if (theme) { - applyTheme(themes[target.dataset.misskeyTheme]); + applyTheme(themes[target.dataset.misskeyTheme as string]); } else { target.removeAttribute('style'); } @@ -97,15 +99,14 @@ const preview = { } else { lastStory = context.id; const channel = addons.getChannel(); - const resetIndexedDBPromise = globalThis.indexedDB?.databases + const resetIndexedDBPromise = (globalThis.indexedDB as IDBFactory | undefined)?.databases ? indexedDB.databases().then((r) => { - for (var i = 0; i < r.length; i++) { + for (let i = 0; i < r.length; i++) { indexedDB.deleteDatabase(r[i].name!); } }).catch(() => {}) : Promise.resolve(); const resetDefaultStorePromise = import('../src/store').then(({ store }) => { - // @ts-expect-error store.init(); }).catch(() => {}); Promise.all([resetIndexedDBPromise, resetDefaultStorePromise]).then(() => { @@ -122,12 +123,12 @@ const preview = { } return story; }, - (Story, context) => { + (_, context) => { return { setup() { return { context, - popups: misskeyOS.popups, + popups: misskeyOS?.popups, }; }, template: diff --git a/packages/frontend/.storybook/themes.d.ts b/packages/frontend/.storybook/themes.d.ts new file mode 100644 index 0000000000..2abf788c31 --- /dev/null +++ b/packages/frontend/.storybook/themes.d.ts @@ -0,0 +1,3 @@ +import type { Theme } from '../src/theme.js'; + +export default {} as Record; diff --git a/packages/frontend/.storybook/tsconfig.json b/packages/frontend/.storybook/tsconfig.json deleted file mode 100644 index 18baf516ba..0000000000 --- a/packages/frontend/.storybook/tsconfig.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "Node16", - "strict": true, - "allowUnusedLabels": false, - "allowUnreachableCode": false, - "exactOptionalPropertyTypes": true, - "noEmitOnError": false, - "noFallthroughCasesInSwitch": true, - "noImplicitOverride": true, - "noImplicitReturns": true, - "noPropertyAccessFromIndexSignature": true, - "noUncheckedIndexedAccess": true, - "noUnusedLocals": true, - "noUnusedParameters": true, - "checkJs": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "incremental": true, - "jsx": "react", - "jsxFactory": "h" - }, - "files": [ - "./changes.ts", - "./generate.tsx", - "./preload-locale.ts", - "./preload-theme.ts" - ] -} diff --git a/packages/frontend/eslint.config.js b/packages/frontend/eslint.config.js index a5381e14e3..f857b9f752 100644 --- a/packages/frontend/eslint.config.js +++ b/packages/frontend/eslint.config.js @@ -16,6 +16,7 @@ export default [ ...pluginVue.configs['flat/recommended'], { files: ['{src,test,js,@types}/**/*.{ts,vue}'], + ignores: ['*.*'], plugins: { sharkey: { rules: { locale: localeRule } } }, languageOptions: { globals: { @@ -43,7 +44,7 @@ export default [ parserOptions: { extraFileExtensions: ['.vue'], parser: tsParser, - project: ['./tsconfig.json'], + project: ['tsconfig.vue.json'], sourceType: 'module', tsconfigRootDir: import.meta.dirname, }, @@ -162,23 +163,120 @@ export default [ autofix: true, }], 'vue/attribute-hyphenation': ['error', 'never'], + 'import/no-default-export': 'off', }, }, { files: ['src/**/*.stories.ts'], rules: { 'no-restricted-globals': 'off', - } + }, + }, + { + files: ['.storybook/**/*.ts', '.storybook/**/*.tsx', '.storybook/**/*.js', '.storybook/**/*.jsx'], + rules: { + 'import/no-default-export': 'off', + 'no-restricted-globals': 'off', + }, + }, + { + files: ['.storybook/**/*.ts', '.storybook/**/*.tsx', '.storybook/**/*.js', '.storybook/**/*.jsx'], + ignores: [ + '.storybook/changes.ts', + '.storybook/main.ts', + '.storybook/generate.tsx', + '.storybook/preload-locale.ts', + '.storybook/preload-theme.ts', + ], + languageOptions: { + parserOptions: { + parser: tsParser, + project: ['tsconfig.vue.storybook.json'], + sourceType: 'module', + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + 'import/no-default-export': 'off', + }, + }, + { + files: [ + '.storybook/changes.ts', + '.storybook/main.ts', + '.storybook/generate.tsx', + '.storybook/preload-locale.ts', + '.storybook/preload-theme.ts', + ], + languageOptions: { + parserOptions: { + parser: tsParser, + project: ['tsconfig.storybook.json'], + sourceType: 'module', + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + 'import/no-default-export': 'off', + }, + }, + { + files: ['test/**/*.ts', 'test/**/*.js'], + languageOptions: { + parserOptions: { + parser: tsParser, + project: ['test/tsconfig.json'], + sourceType: 'module', + tsconfigRootDir: import.meta.dirname, + }, + }, + }, + { + files: [ + '*.js', + '*.ts', + 'lib/**/*.ts', + 'lib/**/*.js', + 'scripts/**/*.ts', + 'scripts/**/*.js', + 'scripts/**/*.mjs', + 'scripts/**/*.cjs', + ], + ignores: [ + 'node_modules', + '.storybook', + 'vue-shims.d.ts', + 'src', + 'test', + '@types', + 'assets', + ], + languageOptions: { + globals: { + ...globals.node, + }, + parserOptions: { + parser: tsParser, + project: ['tsconfig.scripts.json'], + sourceType: 'module', + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + 'import/no-default-export': 'off', + }, }, { ignores: [ - "**/lib/", - "**/temp/", - "**/built/", - "**/coverage/", - "**/node_modules/", - "**/libopenmpt/", - "**/storybook-static/" - ] + '**/lib', + '**/temp', + '**/built', + '**/coverage', + '**/node_modules', + '**/libopenmpt', + '**/storybook-static', + 'vue-shims.d.ts', + 'assets', + ], }, ]; diff --git a/packages/frontend/lib/rollup-plugin-unwind-css-module-class-name.test.ts b/packages/frontend/lib/rollup-plugin-unwind-css-module-class-name.test.ts index ccfa08575b..807a3a5792 100644 --- a/packages/frontend/lib/rollup-plugin-unwind-css-module-class-name.test.ts +++ b/packages/frontend/lib/rollup-plugin-unwind-css-module-class-name.test.ts @@ -171,6 +171,7 @@ const index_photos = /* @__PURE__ */ _export_sfc(_sfc_main, [["__cssModules", cs export { index_photos as default }; `.slice(1), { ecmaVersion: 'latest', sourceType: 'module' }); + // @ts-expect-error This is wrong, but not my problem unwindCssModuleClassName(ast); expect(generate(ast)).toBe(` import {c as api, d as store, i as i18n, aD as notePage, bN as ImgWithBlurhash, bY as getStaticImageUrl, _ as _export_sfc} from './app-!~{001}~.js'; @@ -438,6 +439,7 @@ const MkDateSeparatedList = /* @__PURE__ */ _export_sfc(_sfc_main, [["__cssModul export { MkDateSeparatedList as M }; `.slice(1), { ecmaVersion: 'latest', sourceType: 'module' }); + // @ts-expect-error This is wrong, but not my problem unwindCssModuleClassName(ast); expect(generate(ast)).toBe(` import {a7 as getCurrentInstance, b as defineComponent, G as useCssModule, a1 as h, H as TransitionGroup} from './!~{002}~.js'; diff --git a/packages/frontend/lib/rollup-plugin-unwind-css-module-class-name.ts b/packages/frontend/lib/rollup-plugin-unwind-css-module-class-name.ts index 7ecb1e9179..5302f35e39 100644 --- a/packages/frontend/lib/rollup-plugin-unwind-css-module-class-name.ts +++ b/packages/frontend/lib/rollup-plugin-unwind-css-module-class-name.ts @@ -8,6 +8,7 @@ import { walk } from '../node_modules/estree-walker/src/index.js'; import type * as estree from 'estree'; import type * as estreeWalker from 'estree-walker'; import type { Plugin } from 'vite'; +import type { Identifier } from 'estree'; function isFalsyIdentifier(identifier: estree.Identifier): boolean { return identifier.name === 'undefined' || identifier.name === 'NaN'; @@ -382,7 +383,7 @@ export function unwindCssModuleClassName(ast: estree.Node): void { if (childNode.name !== ident) return; this.replace({ type: 'Identifier', - name: node.declarations[0].id.name, + name: (node.declarations[0].id as Identifier).name, }); }, }); @@ -432,6 +433,7 @@ export function unwindCssModuleClassName(ast: estree.Node): void { type: 'ArrayExpression', elements: node.declarations[0].init.arguments[1].elements.slice(0, __cssModulesIndex).concat(node.declarations[0].init.arguments[1].elements.slice(__cssModulesIndex + 1)), }], + optional: false, }, }], kind: 'const', diff --git a/packages/frontend/lib/vite-plugin-create-search-index.ts b/packages/frontend/lib/vite-plugin-create-search-index.ts index 97f4e589a3..d19cd24620 100644 --- a/packages/frontend/lib/vite-plugin-create-search-index.ts +++ b/packages/frontend/lib/vite-plugin-create-search-index.ts @@ -3,12 +3,10 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -/// - import { parse as vueSfcParse } from 'vue/compiler-sfc'; import { createLogger, - EnvironmentModuleGraph, + type EnvironmentModuleGraph, type LogErrorOptions, type LogOptions, normalizePath, @@ -20,7 +18,7 @@ import { glob } from 'glob'; import JSON5 from 'json5'; import MagicString, { SourceMap } from 'magic-string'; import path from 'node:path' -import { hash, toBase62 } from '../vite.config'; +import { hash, toBase62 } from '../vite.config.js'; import { minimatch } from 'minimatch'; import { type AttributeNode, @@ -63,7 +61,7 @@ interface MarkerRelation { let logger = { info: (msg: string, options?: LogOptions) => { }, warn: (msg: string, options?: LogOptions) => { }, - error: (msg: string, options?: LogErrorOptions | unknown) => { }, + error: (msg: string, options?: LogErrorOptions) => { }, }; let loggerInitialized = false; @@ -470,7 +468,11 @@ export function collectFileMarkers(id: string, code: string): SearchIndexItem[] return extractUsageInfoFromTemplateAst(descriptor.template?.ast, id); } catch (error) { - logger.error(`Error analyzing file ${id}:`, error); + logger.error(`Error analyzing file ${id}:`, { + error: error instanceof Error + ? error + : new Error(`Unknown error of type ${typeof(error)}`, { cause: error }), + }); } return []; diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 0938b26e21..5f29ff1efd 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -3,43 +3,49 @@ "private": true, "type": "module", "scripts": { - "watch": "vite", - "build": "vite build", + "watch": "node scripts/build.mjs --watch", + "build": "node scripts/build.mjs", + "build:pre": "pnpm run -w build-pre && pnpm run --filter misskey-js build && pnpm run --filter misskey-reversi build && pnpm run --filter misskey-bubble-game build && pnpm run --filter sw build && pnpm run --filter frontend-shared build", "storybook-dev": "nodemon --verbose --watch src --ext \"mdx,ts,vue\" --ignore \"*.stories.ts\" --exec \"pnpm build-storybook-pre && pnpm exec storybook dev -p 6006 --ci\"", "build-storybook-pre": "(tsc -p .storybook || echo done.) && node .storybook/generate.js && node .storybook/preload-locale.js && node .storybook/preload-theme.js", "build-storybook": "pnpm build-storybook-pre && storybook build --webpack-stats-json storybook-static", "chromatic": "chromatic", - "test": "vitest --run --globals", - "test-and-coverage": "vitest --run --coverage --globals", - "typecheck": "vue-tsc --noEmit", - "eslint": "eslint --quiet \"{src,test,js,@types}/**/*.{js,jsx,ts,tsx,vue}\" --cache", + "test": "node scripts/vitest.mjs --run --globals", + "test-and-coverage": "node scripts/vitest.mjs --run --coverage --globals", + "typecheck-all": "pnpm run --no-bail typecheck:vue && pnpm run --no-bail typecheck:test && pnpm run --no-bail typecheck:scripts && pnpm run --no-bail typecheck:storybook", + "typecheck": "pnpm run typecheck:vue && pnpm run typecheck:test && pnpm run typecheck:scripts && pnpm run typecheck:storybook", + "typecheck:vue": "vue-tsc -p tsconfig.vue.json --noEmit", + "typecheck:test": "vue-tsc -p test/tsconfig.json --noEmit", + "typecheck:scripts": "tsc -p tsconfig.scripts.json --noEmit", + "typecheck:storybook": "tsc -p tsconfig.storybook.json --noEmit", + "eslint": "eslint --quiet --cache -c eslint.config.js .", "lint": "pnpm typecheck && pnpm eslint" }, "dependencies": { - "@discordapp/twemoji": "15.1.0", + "@discordapp/twemoji": "16.0.1", "@github/webauthn-json": "2.1.1", "@mcaptcha/vanilla-glue": "0.1.0-alpha-3", "@misskey-dev/browser-image-resizer": "2024.1.0", "@phosphor-icons/web": "2.1.2", - "@ruffle-rs/ruffle": "0.1.0-nightly.2024.10.15", - "@sentry/vue": "9.14.0", + "@ruffle-rs/ruffle": "0.2.0-nightly.2025.9.25", + "@sentry/vue": "10.15.0", "@syuilo/aiscript": "0.19.0", "aiscript-vscode": "github:aiscript-dev/aiscript-vscode#v0.1.15", "broadcast-channel": "7.1.0", "buraha": "0.0.1", "canvas-confetti": "1.9.3", - "chart.js": "4.4.9", + "chart.js": "4.5.0", "chartjs-adapter-date-fns": "3.0.0", - "chartjs-chart-matrix": "2.1.1", + "chartjs-chart-matrix": "3.0.0", "chartjs-plugin-gradient": "0.6.1", "chartjs-plugin-zoom": "2.2.0", - "chromatic": "11.28.2", + "chromatic": "13.2.0", "compare-versions": "6.1.1", - "cropperjs": "2.0.0", + "cropperjs": "2.0.1", "date-fns": "4.1.0", "eventemitter3": "5.0.1", "frontend-shared": "workspace:*", - "idb-keyval": "6.2.1", + "idb-keyval": "6.2.2", "insert-text-at-cursor": "0.3.0", "is-file-animated": "1.0.2", "json5": "2.2.3", @@ -52,100 +58,99 @@ "photoswipe": "5.4.4", "promise-limit": "2.7.0", "punycode.js": "2.3.1", - "sanitize-html": "2.16.0", - "shiki": "3.3.0", + "sanitize-html": "2.17.0", + "shiki": "3.13.0", "strict-event-emitter-types": "2.0.0", "textarea-caret": "3.1.0", "throttle-debounce": "5.0.2", "tinycolor2": "1.6.0", - "typescript": "5.8.3", - "uuid": "11.1.0", + "uuid": "13.0.0", "v-code-diff": "1.13.1", - "vue": "3.5.14", + "vue": "3.5.21", "vuedraggable": "next", "wanakana": "5.3.1" }, "optionalDependencies": { - "cypress": "14.3.2" + "cypress": "15.3.0" }, "devDependencies": { - "@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.2", + "@misskey-dev/eslint-plugin": "2.1.0", + "@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.3", "@rollup/plugin-json": "6.1.0", "@rollup/plugin-replace": "6.0.2", - "@rollup/pluginutils": "5.1.4", - "@storybook/addon-actions": "8.6.12", - "@storybook/addon-essentials": "8.6.12", - "@storybook/addon-interactions": "8.6.12", - "@storybook/addon-links": "8.6.12", - "@storybook/addon-mdx-gfm": "8.6.12", - "@storybook/addon-storysource": "8.6.12", - "@storybook/blocks": "8.6.12", - "@storybook/components": "8.6.12", - "@storybook/core-events": "8.6.12", - "@storybook/manager-api": "8.6.12", - "@storybook/preview-api": "8.6.12", - "@storybook/react": "8.6.12", - "@storybook/react-vite": "8.6.12", - "@storybook/test": "8.6.12", - "@storybook/theming": "8.6.12", - "@storybook/types": "8.6.12", - "@storybook/vue3": "8.6.12", - "@storybook/vue3-vite": "8.6.12", + "@rollup/pluginutils": "5.3.0", + "@storybook/addon-essentials": "8.6.14", + "@storybook/addon-interactions": "8.6.14", + "@storybook/addon-links": "9.1.8", + "@storybook/addon-mdx-gfm": "8.6.14", + "@storybook/addon-storysource": "8.6.14", + "@storybook/blocks": "8.6.14", + "@storybook/components": "8.6.14", + "@storybook/core-events": "8.6.14", + "@storybook/manager-api": "8.6.14", + "@storybook/preview-api": "8.6.14", + "@storybook/react": "9.1.8", + "@storybook/react-vite": "9.1.8", + "@storybook/test": "8.6.14", + "@storybook/theming": "8.6.14", + "@storybook/types": "8.6.14", + "@storybook/vue3": "9.1.8", + "@storybook/vue3-vite": "9.1.8", "@testing-library/vue": "8.1.0", - "@twemoji/parser": "15.1.1", "@types/canvas-confetti": "1.9.0", - "@types/estree": "1.0.7", + "@types/estree": "1.0.8", "@types/katex": "0.16.7", - "@types/matter-js": "0.19.8", + "@types/matter-js": "0.20.2", "@types/micromatch": "4.0.9", - "@types/node": "22.15.2", + "@types/node": "22.18.1", "@types/punycode.js": "npm:@types/punycode@2.1.4", - "@types/sanitize-html": "2.15.0", + "@types/sanitize-html": "2.16.0", "@types/seedrandom": "3.0.8", "@types/throttle-debounce": "5.0.2", "@types/tinycolor2": "1.4.6", "@types/ws": "8.18.1", - "@typescript-eslint/eslint-plugin": "8.31.0", - "@typescript-eslint/parser": "8.31.0", - "@vitejs/plugin-vue": "5.2.3", - "@vitest/coverage-v8": "3.1.2", - "@vue/compiler-core": "3.5.14", - "@vue/compiler-sfc": "3.5.14", - "@vue/runtime-core": "3.5.14", - "mfm-js": "npm:@transfem-org/sfm-js@0.24.8", - "acorn": "8.14.1", + "@typescript-eslint/eslint-plugin": "8.44.1", + "@typescript-eslint/parser": "8.44.1", + "@vitejs/plugin-vue": "6.0.1", + "@vitest/coverage-v8": "3.2.4", + "@vue/compiler-core": "3.5.21", + "@vue/runtime-core": "3.5.21", + "acorn": "8.15.0", "astring": "1.9.0", - "cross-env": "7.0.3", - "eslint-plugin-import": "2.31.0", - "eslint-plugin-vue": "10.0.0", + "cross-env": "10.0.0", + "eslint-plugin-import": "2.32.0", + "eslint-plugin-vue": "10.5.0", "estree-walker": "3.0.3", + "execa": "9.6.0", "fast-glob": "3.3.3", - "happy-dom": "17.4.4", + "happy-dom": "18.0.1", "intersection-observer": "0.12.2", - "magic-string": "0.30.17", + "magic-string": "0.30.19", + "mfm-js": "npm:@transfem-org/sfm-js@0.26.1", "micromatch": "4.0.8", - "minimatch": "10.0.1", - "msw": "2.7.5", - "msw-storybook-addon": "2.0.4", + "minimatch": "10.0.3", + "msw": "2.11.3", + "msw-storybook-addon": "2.0.5", "nodemon": "3.1.10", - "prettier": "3.5.3", - "react": "19.1.0", - "react-dom": "19.1.0", - "rollup": "4.40.0", - "sass": "1.87.0", + "prettier": "3.6.2", + "react": "19.1.1", + "react-dom": "19.1.1", + "rollup": "4.52.2", + "sass": "1.93.2", "seedrandom": "3.0.5", - "start-server-and-test": "2.0.11", - "storybook": "8.6.12", + "start-server-and-test": "2.1.2", + "storybook": "9.1.8", "storybook-addon-misskey-theme": "github:misskey-dev/storybook-addon-misskey-theme", - "three": "0.176.0", - "tsc-alias": "1.8.15", + "three": "0.180.0", + "tsc-alias": "1.8.16", "tsconfig-paths": "4.2.0", - "vite": "6.3.4", + "typescript": "5.9.2", + "vite": "7.1.7", "vite-plugin-turbosnap": "1.0.3", - "vitest": "3.1.2", + "vitest": "3.2.4", "vitest-fetch-mock": "0.4.5", - "vue-component-type-helpers": "2.2.10", - "vue-eslint-parser": "10.1.3", - "vue-tsc": "2.2.10" + "vue-component-type-helpers": "3.0.8", + "vue-eslint-parser": "10.2.0", + "vue-tsc": "3.0.8" } } diff --git a/packages/frontend/scripts/build.mjs b/packages/frontend/scripts/build.mjs new file mode 100644 index 0000000000..2f0883d3cc --- /dev/null +++ b/packages/frontend/scripts/build.mjs @@ -0,0 +1,37 @@ +/** + * Hot-swaps tsconfig files to work around vite limitations. + * Based on idea from https://github.com/vitejs/vite/discussions/8483#discussioncomment-6830634 + */ + +import nodeFs from 'node:fs/promises'; +import nodePath from 'node:path'; +import { execa } from 'execa'; + +const rootDir = nodePath.resolve(import.meta.dirname, '../'); +const tsConfig = nodePath.resolve(rootDir, 'tsconfig.json'); +const tsConfigBak = nodePath.resolve(rootDir, 'tsconfig.json.bak'); +const tsConfigVue = nodePath.resolve(rootDir, 'tsconfig.vue.json'); + +const mode = process.argv.slice(2).includes('--watch') ? 'watch' : 'build'; + +console.log('Staging tsconfig.vue.json as tsconfig.json...'); +await nodeFs.rename(tsConfig, tsConfigBak); +await nodeFs.copyFile(tsConfigVue, tsConfig); + +try { + console.log('Starting vite...'); + await execa( + 'vite', + mode === 'build' + ? ['build'] + : [], + { + stdout: process.stdout, + stderr: process.stderr, + }, + ); +} finally { + console.log('Restoring original tsconfig.json...'); + await nodeFs.rm(tsConfig); + await nodeFs.rename(tsConfigBak, tsConfig); +} diff --git a/packages/frontend/scripts/vitest.mjs b/packages/frontend/scripts/vitest.mjs new file mode 100644 index 0000000000..33d32943f3 --- /dev/null +++ b/packages/frontend/scripts/vitest.mjs @@ -0,0 +1,33 @@ +/** + * Hot-swaps tsconfig files to work around vite limitations. + * Based on idea from https://github.com/vitejs/vite/discussions/8483#discussioncomment-6830634 + */ + +import nodeFs from 'node:fs/promises'; +import nodePath from 'node:path'; +import { execa } from 'execa'; + +const rootDir = nodePath.resolve(import.meta.dirname, '../'); +const tsConfig = nodePath.resolve(rootDir, 'tsconfig.json'); +const tsConfigBak = nodePath.resolve(rootDir, 'tsconfig.json.bak'); +const tsConfigVue = nodePath.resolve(rootDir, 'tsconfig.vue.json'); + +console.log('Staging tsconfig.vue.json as tsconfig.json...'); +await nodeFs.rename(tsConfig, tsConfigBak); +await nodeFs.copyFile(tsConfigVue, tsConfig); + +try { + console.log('Starting vitest...'); + await execa( + 'vitest', + process.argv.slice(2), + { + stdout: process.stdout, + stderr: process.stderr, + }, + ); +} finally { + console.log('Restoring original tsconfig.json...'); + await nodeFs.rm(tsConfig); + await nodeFs.rename(tsConfigBak, tsConfig); +} diff --git a/packages/frontend/src/components/MkEmojiPicker.vue b/packages/frontend/src/components/MkEmojiPicker.vue index cf7087d798..657044d979 100644 --- a/packages/frontend/src/components/MkEmojiPicker.vue +++ b/packages/frontend/src/components/MkEmojiPicker.vue @@ -150,6 +150,9 @@ const props = withDefaults(defineProps<{ targetNote?: Misskey.entities.Note; }>(), { showPinned: true, + pinnedEmojis: undefined, + maxHeight: undefined, + targetNote: undefined, }); const emit = defineEmits<{ @@ -184,37 +187,41 @@ const searchResultCustom = ref([]); const searchResultUnicode = ref([]); const tab = ref<'index' | 'custom' | 'unicode' | 'tags'>('index'); -const customEmojiFolderRoot: CustomEmojiFolderTree = { value: '', category: '', children: [] }; +const customEmojiFolderRoot = computed(() => { + const root = { value: '', category: '', children: [] }; -function parseAndMergeCategories(input: string, root: CustomEmojiFolderTree): CustomEmojiFolderTree { - const parts = input.split('/').map(p => p.trim()); - let currentNode: CustomEmojiFolderTree = root; + function parseAndMergeCategories(category: string): CustomEmojiFolderTree { + const parts = category.split('/').map(p => p.trim()); + let currentNode: CustomEmojiFolderTree = root; - const currentPath = [] as string[]; - for (const part of parts) { - currentPath.push(part); - let existingNode = currentNode.children.find((node) => node.value === part); + const currentPath = [] as string[]; + for (const part of parts) { + currentPath.push(part); + let existingNode = currentNode.children.find((node) => node.value === part); - if (!existingNode) { - const newNode: CustomEmojiFolderTree = { value: part, category: currentPath.join("/"), children: [] }; - currentNode.children.push(newNode); - existingNode = newNode; + if (!existingNode) { + const newNode: CustomEmojiFolderTree = { value: part, category: currentPath.join("/"), children: [] }; + currentNode.children.push(newNode); + existingNode = newNode; + } + + currentNode = existingNode; } - currentNode = existingNode; + return currentNode; } - return currentNode; -} + customEmojiCategories.value.forEach(ec => { + if (ec !== null) { + parseAndMergeCategories(ec); + } + }); -customEmojiCategories.value.forEach(ec => { - if (ec !== null) { - parseAndMergeCategories(ec, customEmojiFolderRoot); - } + parseAndMergeCategories(''); + + return root; }); -parseAndMergeCategories('', customEmojiFolderRoot); - watch(q, () => { if (emojisEl.value) emojisEl.value.scrollTop = 0; diff --git a/packages/frontend/src/components/MkPostFormDialog.vue b/packages/frontend/src/components/MkPostFormDialog.vue index aa3eebb257..8f0b629468 100644 --- a/packages/frontend/src/components/MkPostFormDialog.vue +++ b/packages/frontend/src/components/MkPostFormDialog.vue @@ -27,17 +27,17 @@ SPDX-License-Identifier: AGPL-3.0-only