misty.cafe/mistykey
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

merge: Deterministic operation (!1235)

Browse source 
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/1235

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Marie <github@yuugi.dev>
This commit is contained in:
Hazelnoot 2025-11-12 17:04:17 -05:00
commit 90201307a0
777 changed files with 73731 additions and 51614 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.config/ci.yml
View file

@ -374,9 +374,6 @@ attachLdSignatureForRelays: true
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
# # default: false
# disableQueryTruncation: false
# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable.
# # default: false in production, true otherwise.
# #verbose: false
# Settings for the activity logger, which records inbound activities to the database.
# Disabled by default due to the large volume of data it saves.

3
.config/cypress-devcontainer.yml
View file

@ -325,9 +325,6 @@ allowedPrivateNetworks:
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
# # default: false
# disableQueryTruncation: false
# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable.
# # default: false in production, true otherwise.
# #verbose: false
# Settings for the activity logger, which records inbound activities to the database.
# Disabled by default due to the large volume of data it saves.

3
.config/docker_example.yml
View file

@ -442,9 +442,6 @@ attachLdSignatureForRelays: true
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
# # default: false
# disableQueryTruncation: false
# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable.
# # default: false in production, true otherwise.
# #verbose: false
# Settings for the activity logger, which records inbound activities to the database.
# Disabled by default due to the large volume of data it saves.

3
.config/example.yml
View file

@ -448,9 +448,6 @@ attachLdSignatureForRelays: true
# # Disable query truncation. If set to true, the full text of the query will be output to the log.
# # default: false
# disableQueryTruncation: false
# # Shows debug log messages after instance startup. To capture earlier debug logs, set the MK_VERBOSE environment variable.
# # default: false in production, true otherwise.
# #verbose: false
# Settings for the activity logger, which records inbound activities to the database.
# Disabled by default due to the large volume of data it saves.

36
.gitlab-ci.yml
View file

@ -128,13 +128,10 @@ backend_tests:
- name: redis
pull_policy: if-not-present
script:
- >-
pnpm run build \
--filter=backend \
--filter=megalodon \
--filter=misskey-js
- pnpm run migrate
- pnpm run test --filter=backend
- pnpm run --filter backend build:pre
- pnpm run --filter backend build
- pnpm run --filter backend migrate
- pnpm run --filter backend test
# Same as common, but MRs are only run if they modify the backend.
rules:
- if: $CI_PIPELINE_SOURCE == 'push' && ($CI_COMMIT_BRANCH == 'develop' || $CI_COMMIT_BRANCH == 'stable')
@ -152,16 +149,9 @@ backend_tests:
frontend_tests:
<<: *test_common
script:
- >-
pnpm run build \
--filter=frontend \
--filter=frontend-embed \
--filter=frontend-shared \
--filter=misskey-js \
--filter=misskey-bubble-game \
--filter=misskey-reversi \
--filter=sw
- pnpm run test --filter=frontend --filter=misskey-js
- pnpm run --filter frontend build:pre
- pnpm run --filter frontend build
- pnpm run --filter frontend test
# Same as common, but MRs are only run if they modify the frontend.
rules:
- if: $CI_PIPELINE_SOURCE == 'push' && ($CI_COMMIT_BRANCH == 'develop' || $CI_COMMIT_BRANCH == 'stable')
@ -182,6 +172,18 @@ frontend_tests:
- 'cypress/**/*'
- 'assets/**/*'
misskey-js_tests:
<<: *test_common
script:
- pnpm run --filter misskey-js build
- pnpm run --filter misskey-js test
megalodon_tests:
<<: *test_common
script:
- pnpm run --filter megalodon build
- pnpm run --filter megalodon test
get_image_tag:
<<: *deploy_common
image:

60
CONTRIBUTING.Sharkey.md
View file

@ -326,11 +326,14 @@ git checkout -m merge/$(date +%Y-%m-%d) # Create/switch to a merge branch for
git merge --no-ff misskey/develop # Merge from Misskey's develop branch, forcing a merge commit.
```
Fix conflicts and *commit!* Conflicts in `pnpm-lock.yaml` can usually
be fixed by running `pnpm install` - it detects conflict markers and
seems to do a decent job.
Fix conflicts and *commit!*
- Conflicts in `pnpm-lock.yaml` can be fixed by rejecting changes and running `pnpm install`.
- Conflicts in `packages/misskey-js/etc` and `packages/misskey-js/src/autogen` can be fixed by rejecting changes and running `pnpm run build-misskey-js-with-types`.
- Conflicts in `locales/index.d.ts` can be fixed by rejecting changes and running `pnpm run build-assets`.
- Conflicts in any `package.json` file can be fixed by merging only added/removed dependencies, then running `pnpm run sync-dependency-versions`. Other changes (not dependencies) will need to be merged manually.
- Conflicts involving `this.timeService.now` or `this.timeService.date` can be resolved by accepting remote changes. ESLint will highlight all the missing references in a later step.
*After that commit,* do all the extra work, on the same branch:
*After that commit*, do all the extra work on the same branch:
- Copy all changes (commit after each step):
- in `packages/backend/src/core/activitypub/models/ApNoteService.ts`, from `createNote` to `updateNote`
@ -358,14 +361,44 @@ seems to do a decent job.
- from `.config/example.yml` to `.config/ci.yml` and `chart/files/default.yml`
- in `packages/backend/src/core/MfmService.ts`, from `toHtml` to `toMastoApiHtml`
- from `verifyLink` in `packages/backend/src/core/activitypub/models/ApPersonService.ts` to `verifyFieldLinks` in `packages/backend/src/misc/verify-field-link.ts` (if sensible)
- If there have been any changes to the federated user data (the
`renderPerson` function in
`packages/backend/src/core/activitypub/ApRendererService.ts`), make
sure that the set of fields in `userNeedsPublishing` and
`profileNeedsPublishing` in
`packages/backend/src/server/api/endpoints/i/update.ts` are still
correct.
- Check for changes that may require additional work:
- If there have been any changes to the federated user data (the
`renderPerson` function in
`packages/backend/src/core/activitypub/ApRendererService.ts`), make
sure that the set of fields in `userNeedsPublishing` and
`profileNeedsPublishing` in
`packages/backend/src/server/api/endpoints/i/update.ts` are still
correct.
- Check for any new instances of any memory cache class.
(`MemoryKVCache`, `MemorySingleCache`, `RedisKVCache`, `RedisSingleCache`, and `QuantumKVCache` are the current ones.)
These can usually be kept as-is, but all instances must be managed by `CacheManagementService`.
The conversion is easy:
1. Make sure that `CacheManagementService` is available.
In most cases, it can be injected through DI.
(it's in the `GlobalModule` which should be available everywhere.)
2. Find where the cache is constructed.
If it's a field initializer, then move it to the constructor (splitting declaration and initialization.)
3. Replace the `new Whatever()` statement with a call to `cacheManagementService.createWhatever()`.
Arguments can be kept as-is, but remove any references to `Redis`, `InternalEventService`, or `TimeService`.
(these are provided by `CacheManagementService` directly.)
4. Remove any calls to `dispose()` the cache.
Disposal is managed by `CacheManagementService`, so attempting to call any `dispose` or `onApplicationShutdown` method will produce a type error.
- Check for any new calls to native time functions:
- `Date.now()` - replace with `this.timeService.now`.
Inject `TimeService` via DI if it's not already available.
- `new Date()` - if there's a value passed in, then leave it.
But the no-args constructor should be replaced with `this.timeService.date`.
Inject `TimeService` via DI if it's not already available.
- `setTimeout` - migrate to `this.timeService.startTimer` or `this.timeService.startPromiseTimer`.
The parameters should be the same, but the return type is different.
You may need to replace some `NodeJS.Timeout` types with `this.timerHandle`.
Inject `TimeService` via DI if it's not already available.
- `setInterval` - migrate to `this.timeService.startTimer`.
Migration is mostly the same as `setTimeout`, but with one major difference:
You must add `{ repeated: true }` as the final option parameter.
If this is omitted, the code will compile but the interval will only fire once!
- Check for any new Chart subclasses, and make sure to inject `TimeService` and implement `getCurrentDate`.
- Check for any new Channel subclasses and add all missing DI parameters.
- Check the changes against our `develop` branch (`git diff develop`)
and against Misskey's `develop` branch (`git diff misskey/develop`).
@ -465,3 +498,6 @@ following apply:
together. Using `MemorySingleCache` or `RedisSingleCache` could
provide a cleaner implementation without resorting to hacks like a
fixed key.
- It's necessary to use `null` as a data value.
`QuantumKVCache` does not allow null values, and thus another option should be chosen.

5
cypress/tsconfig.json
View file

@ -1,9 +1,8 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "../packages/shared/tsconfig.web.jsonc",
"compilerOptions": {
"lib": ["dom", "es5"],
"target": "es5",
"types": ["cypress", "node"],
"incremental": true
},
"include": ["./**/*.ts"]
}

4
locales/package.json
View file

@ -1,3 +1,5 @@
{
"type": "module"
"type": "module",
"main": "index.js",
"types": "index.d.ts"
}

46
package.json
View file

@ -6,7 +6,7 @@
"type": "git",
"url": "https://activitypub.software/TransFem-org/Sharkey.git"
},
"packageManager": "pnpm@9.6.0",
"packageManager": "pnpm@10.16.0",
"workspaces": [
"packages/frontend-shared",
"packages/frontend",
@ -36,7 +36,7 @@
"lint": "pnpm -r lint",
"lint-all": "pnpm -r --no-bail lint",
"eslint": "pnpm -r eslint",
"eslint-all": "pnpm -r --no-bail eslint",
"eslint-all": "pnpm -r --no-bail eslint-all && pnpm -r --no-bail eslint",
"cy:open": "pnpm cypress open --browser --e2e --config-file=cypress.config.ts",
"cy:run": "pnpm cypress run",
"e2e": "pnpm start-server-and-test start:test http://localhost:61812 cy:run",
@ -47,38 +47,40 @@
"test-and-coverage": "pnpm -r test-and-coverage",
"clean": "node ./scripts/clean.js",
"clean-all": "node ./scripts/clean-all.js",
"cleanall": "pnpm clean-all"
"cleanall": "pnpm clean-all",
"sync-dependency-versions": "node scripts/sync-deps.mjs"
},
"resolutions": {
"chokidar": "4.0.3",
"lodash": "4.17.21"
"lodash": "4.17.21",
"axios": "1.12.2"
},
"dependencies": {
"js-yaml": "4.1.0"
},
"optionalDependencies": {
"cypress": "14.3.2"
"cypress": "15.3.0"
},
"devDependencies": {
"@misskey-dev/eslint-plugin": "2.1.0",
"@types/node": "22.15.2",
"@typescript-eslint/eslint-plugin": "8.31.0",
"@typescript-eslint/parser": "8.31.0",
"cross-env": "7.0.3",
"cssnano": "7.0.6",
"esbuild": "0.25.3",
"eslint": "9.25.1",
"execa": "9.5.2",
"@types/node": "22.18.1",
"@typescript-eslint/eslint-plugin": "8.44.1",
"@typescript-eslint/parser": "8.44.1",
"cross-env": "10.0.0",
"cssnano": "7.1.1",
"esbuild": "0.25.10",
"eslint": "9.36.0",
"execa": "9.6.0",
"fast-glob": "3.3.3",
"glob": "11.0.2",
"globals": "16.1.0",
"glob": "11.0.3",
"globals": "16.4.0",
"ignore-walk": "8.0.0",
"ncp": "2.0.0",
"pnpm": "9.6.0",
"ignore-walk": "7.0.0",
"postcss": "8.5.3",
"start-server-and-test": "2.0.11",
"tar": "7.4.3",
"terser": "5.39.0",
"typescript": "5.8.3"
"pnpm": "10.17.1",
"postcss": "8.5.6",
"start-server-and-test": "2.1.2",
"tar": "7.5.1",
"terser": "5.44.0",
"typescript": "5.9.2"
}
}

2
packages/backend/.swcrc
View file

@ -17,7 +17,7 @@
"paths": {
"@/*": ["*"]
},
"target": "es2022"
"target": "ESNext"
},
"minify": false,
"sourceMaps": "inline"

2
packages/backend/assets/embed.js
View file

@ -7,7 +7,7 @@
/** @type {NodeListOf<HTMLIFrameElement>} */
const els = document.querySelectorAll('iframe[data-misskey-embed-id]');
window.addEventListener('message', function (event) {
window.addEventListener('message', (event) => {
els.forEach((el) => {
if (event.source !== el.contentWindow) {
return;

150
packages/backend/eslint.config.js
View file

@ -5,7 +5,20 @@ import sharedConfig from '../shared/eslint.config.js';
export default [
...sharedConfig,
{
ignores: ['**/node_modules', 'built', '@types/**/*', 'migration'],
ignores: [
'**/built/',
'migration/',
'**/node_modules/',
'test/',
'test-federation/',
'test-server/',
'**/temp/',
'**/@types/',
'**/coverage/',
'ormconfig.js',
'scripts/check_connect.js',
'scripts/generate_api_json.js',
],
},
{
languageOptions: {
@ -15,11 +28,15 @@ export default [
},
},
{
files: ['**/*.ts', '**/*.tsx'],
files: ['src/**/*.ts', 'src/**/*.tsx'],
ignores: [
'*.*',
'src/server/web/**/*.d.ts',
],
languageOptions: {
parserOptions: {
parser: tsParser,
project: ['./tsconfig.json', './test/tsconfig.json', './test-federation/tsconfig.json'],
project: ['./tsconfig.backend.json'],
sourceType: 'module',
tsconfigRootDir: import.meta.dirname,
},
@ -42,17 +59,89 @@ export default [
position: 'after',
}],
}],
'no-restricted-globals': ['error', {
name: '__dirname',
message: 'Not in ESModule. Use `import.meta.url` instead.',
}, {
name: '__filename',
message: 'Not in ESModule. Use `import.meta.url` instead.',
}],
'no-restricted-globals': [
'error',
{
globals: [
{
name: '__dirname',
message: 'Not in ESModule. Use `import.meta.url` instead.',
},
{
name: '__filename',
message: 'Not in ESModule. Use `import.meta.url` instead.',
},
{
name: 'setTimeout',
message: 'Use TimeService.startTimer instead.',
},
{
name: 'setInterval',
message: 'Use TimeService.startTimer instead.',
},
{
name: 'console',
message: 'Use a Logger instance instead.',
},
],
checkGlobalObject: true,
},
],
'no-restricted-properties': [
'error',
{
object: 'Date',
property: 'now',
message: 'Use TimeService.now instead.',
},
],
'no-restricted-syntax': [
'error',
{
'selector': 'NewExpression[callee.name=\'Date\'][arguments.length=0]',
'message': 'new Date() is restricted. Use TimeService.date instead.',
},
{
'selector': 'NewExpression[callee.name=\'MemoryKVCache\']',
'message': 'Cache constructor will produce an unmanaged instance. Use CacheManagementService.createMemoryKVCache() instead.',
},
{
'selector': 'NewExpression[callee.name=\'MemorySingleCache\']',
'message': 'Cache constructor will produce an unmanaged instance. Use CacheManagementService.createMemorySingleCache() instead.',
},
{
'selector': 'NewExpression[callee.name=\'RedisKVCache\']',
'message': 'Cache constructor will produce an unmanaged instance. Use CacheManagementService.createRedisKVCache() instead.',
},
{
'selector': 'NewExpression[callee.name=\'RedisSingleCache\']',
'message': 'Cache constructor will produce an unmanaged instance. Use CacheManagementService.createRedisSingleCache() instead.',
},
{
'selector': 'NewExpression[callee.name=\'QuantumKVCache\']',
'message': 'Cache constructor will produce an unmanaged instance. Use CacheManagementService.createQuantumKVCache() instead.',
},
{
'selector': 'CallExpression[callee.property.name=\'delete\'][arguments.length=1] > ObjectExpression[properties.length=0]',
'message': 'repository.delete({}) will produce a runtime error. Use repository.deleteAll() instead.',
},
{
'selector': 'CallExpression[callee.property.name=\'update\'][arguments.length>=1] > ObjectExpression[properties.length=0]',
'message': 'repository.update({}, {...}) will produce a runtime error. Use repository.updateAll({...}) instead.',
},
],
},
},
{
files: ['src/server/web/**/*.js', 'src/server/web/**/*.ts'],
files: [
'./assets/**/*.js',
'./assets/**/*.mjs',
'./assets/**/*.cjs',
'./src/server/web/**/*.js',
'./src/server/web/**/*.mjs',
'./src/server/web/**/*.cjs',
'./src/server/web/**/*.d.ts',
],
languageOptions: {
globals: {
...globals.browser,
@ -60,16 +149,41 @@ export default [
CLIENT_ENTRY: true,
LANGS_VERSION: true,
},
parserOptions: {
parser: tsParser,
project: ['./tsconfig.frontend.json'],
sourceType: 'module',
tsconfigRootDir: import.meta.dirname,
},
},
rules: {
'no-restricted-globals': 'off',
'no-restricted-properties': 'off',
'no-restricted-syntax': 'off',
},
},
{
files: [
'eslint.*',
'jest.*',
'scripts/dev.mjs',
'scripts/watch.mjs',
],
ignores: [
"**/lib/",
"**/temp/",
"**/built/",
"**/coverage/",
"**/node_modules/",
"**/migration/",
]
'ormconfig.js',
'scripts/check_connect.js',
'scripts/generate_api_json.js',
],
languageOptions: {
parserOptions: {
parser: tsParser,
project: ['./tsconfig.scripts.json'],
sourceType: 'module',
tsconfigRootDir: import.meta.dirname,
},
},
rules: {
'import/no-default-export': 'off',
},
},
];

15
packages/backend/jest.config.cjs → packages/backend/jest.config.common.ts
View file

@ -3,7 +3,7 @@
* https://jestjs.io/docs/en/configuration.html
*/
module.exports = {
export default {
// All imported modules in your tests should be mocked automatically
// automock: false,
@ -91,7 +91,7 @@ module.exports = {
// See https://github.com/swc-project/jest/issues/64#issuecomment-1029753225
// TODO: Use `--allowImportingTsExtensions` on TypeScript 5.0 so that we can
// directly import `.ts` files without this hack.
'^((?:\\.{1,2}|[A-Z:])*/.*)\\.js$': '$1',
'^(\\.{1,2}[\\/\\\\].*)\\.js$': '$1',
},
// An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
@ -129,7 +129,9 @@ module.exports = {
// A list of paths to directories that Jest should use to search for files in
roots: [
"<rootDir>"
'<rootDir>/src',
'<rootDir>/test',
'<rootDir>/test-federation',
],
// Allows you to use a custom runner instead of Jest's default test runner
@ -142,7 +144,7 @@ module.exports = {
// setupFilesAfterEnv: [],
// The number of seconds after which a test is considered as slow and reported as such in the results.
// slowTestThreshold: 5,
slowTestThreshold: 5,
// A list of paths to snapshot serializer modules Jest should use for snapshot testing
// snapshotSerializers: [],
@ -157,10 +159,7 @@ module.exports = {
// testLocationInResults: false,
// The glob patterns Jest uses to detect test files
testMatch: [
"<rootDir>/test/unit/**/*.ts",
"<rootDir>/src/**/*.test.ts",
],
testMatch: [],
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
// testPathIgnorePatterns: [

6
packages/backend/jest.config.e2e.cjs → packages/backend/jest.config.e2e.ts
View file

@ -3,12 +3,12 @@
* https://jestjs.io/docs/en/configuration.html
*/
const base = require('./jest.config.cjs')
import base from './jest.config.common.ts';
module.exports = {
export default {
...base,
globalSetup: "<rootDir>/built-test/entry.js",
setupFilesAfterEnv: ["<rootDir>/test/jest.setup.ts"],
setupFilesAfterEnv: ["<rootDir>/test/jest.setup.e2e.mjs"],
testMatch: [
"<rootDir>/test/e2e/**/*.ts",
],

2
packages/backend/jest.config.fed.cjs → packages/backend/jest.config.fed.ts
View file

@ -3,7 +3,7 @@
* https://jestjs.io/docs/en/configuration.html
*/
const base = require('./jest.config.cjs');
import base from './jest.config.common.ts';
module.exports = {
...base,

5
packages/backend/jest.config.unit.cjs → packages/backend/jest.config.unit.ts
View file

@ -3,10 +3,11 @@
* https://jestjs.io/docs/en/configuration.html
*/
const base = require('./jest.config.cjs')
import base from './jest.config.common.ts';
module.exports = {
export default {
...base,
globalSetup: '<rootDir>/test/jest.setup.unit.mjs',
testMatch: [
"<rootDir>/test/unit/**/*.ts",
"<rootDir>/src/**/*.test.ts",

6
packages/backend/jest.js
View file

@ -24,7 +24,11 @@ child.on('error', (err) => {
});
child.on('exit', (code, signal) => {
if (code === null) {
process.exit(128 + signal);
if (signal != null) {
process.exit(128 + signal);
} else {
process.exit(128);
}
} else {
process.exit(code);
}

13
packages/backend/jsconfig.json
View file

@ -1,13 +0,0 @@
{
"compilerOptions": {
"target": "es6",
"module": "commonjs",
"allowSyntheticDefaultImports": true
},
"exclude": [
"node_modules",
"jspm_packages",
"tmp",
"temp"
]
}

6
packages/backend/migration/1740162088574-add_unsignedFetch.js
View file

@ -1,4 +1,7 @@
import { loadConfig } from '../built/config.js';
import { LoggerService } from '../built/core/LoggerService.js';
import { NativeTimeService } from '../built/global/TimeService.js';
import { EnvService } from '../built/global/EnvService.js';
export class AddUnsignedFetch1740162088574 {
name = 'AddUnsignedFetch1740162088574'
@ -16,7 +19,8 @@ export class AddUnsignedFetch1740162088574 {
await queryRunner.query(`UPDATE "user" SET "allowUnsignedFetch" = 'always' WHERE "username" LIKE '%.%' AND "host" IS null`);
// Special one-time migration: convert legacy config "" to meta setting ""
const config = await loadConfig();
const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService());
const config = await loadConfig(loggerService);
if (config.checkActivityPubGetSignature) {
// noinspection SqlWithoutWhere
await queryRunner.query(`UPDATE "meta" SET "allowUnsignedFetch" = 'never'`);

6
packages/backend/ormconfig.js
View file

@ -2,8 +2,12 @@ import { DataSource } from 'typeorm';
import { loadConfig } from './built/config.js';
import { entities } from './built/postgres.js';
import { isConcurrentIndexMigrationEnabled } from "./migration/js/migration-config.js";
import { LoggerService } from './built/core/LoggerService.js';
import { NativeTimeService } from './built/global/TimeService.js';
import { EnvService } from './built/global/EnvService.js';
const config = loadConfig();
const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService());
const config = loadConfig(loggerService);
export default new DataSource({
type: 'postgres',

237
packages/backend/package.json
View file

@ -16,41 +16,53 @@
"revert": "pnpm typeorm migration:revert -d ormconfig.js",
"check:connect": "node ./scripts/check_connect.js",
"build": "swc src -d built -D --strip-leading-paths",
"build:pre": "pnpm run -w build-pre && pnpm run --filter megalodon build && pnpm run --filter misskey-js build && pnpm run --filter misskey-reversi build && pnpm run --filter misskey-bubble-game build && pnpm run -w build-assets",
"build:test": "swc test-server -d built-test -D --config-file test-server/.swcrc --strip-leading-paths",
"watch:swc": "swc src -d built -D -w --strip-leading-paths",
"build:tsc": "tsc -p tsconfig.json && tsc-alias -p tsconfig.json",
"watch": "node ./scripts/watch.mjs",
"build:tsc": "tsc -p tsconfig.backend.json && tsc-alias -p tsconfig.backend.json",
"watch": "pnpm run build:pre && node ./scripts/watch.mjs",
"restart": "pnpm build && pnpm start",
"dev": "node ./scripts/dev.mjs",
"typecheck": "pnpm --filter megalodon build && tsc --noEmit && tsc -p test --noEmit && tsc -p test-federation --noEmit",
"eslint": "eslint --quiet \"{src,test-federation,test,js,@types}/**/*.{js,jsx,ts,tsx,vue}\" --cache",
"dev": "pnpm run build:pre && node ./scripts/dev.mjs",
"typecheck-all": "pnpm run --no-bail build:pre && pnpm run --no-bail typecheck:scripts && pnpm run --no-bail typecheck:backend && pnpm run --no-bail typecheck:frontend && pnpm run --no-bail typecheck:test && pnpm run --no-bail typecheck:test-federation && pnpm run --no-bail typecheck:test-server",
"typecheck": "pnpm run build:pre && pnpm run typecheck:scripts && pnpm run typecheck:backend && pnpm run typecheck:frontend && pnpm run typecheck:test && pnpm run typecheck:test-federation && pnpm run typecheck:test-server",
"typecheck:scripts": "tsc -p tsconfig.scripts.json --noEmit",
"typecheck:backend": "tsc -p tsconfig.backend.json --noEmit",
"typecheck:frontend": "tsc -p tsconfig.frontend.json --noEmit",
"typecheck:test": "tsc -p test/tsconfig.json --noEmit",
"typecheck:test-federation": "tsc -p test-federation/tsconfig.json --noEmit",
"typecheck:test-server": "tsc -p test-server/tsconfig.json --noEmit",
"eslint-all": "pnpm run --no-bail eslint:backend && pnpm run --no-bail eslint:test && pnpm run --no-bail eslint:test-federation && pnpm run --no-bail eslint:test-server",
"eslint": "pnpm run eslint:backend && pnpm run eslint:test && pnpm run eslint:test-federation && pnpm run eslint:test-server",
"eslint:backend": "eslint --quiet --cache -c eslint.config.js .",
"eslint:test": "eslint --quiet --cache -c test/eslint.config.js ./test",
"eslint:test-federation": "eslint --quiet --cache -c test-federation/eslint.config.js ./test-federation",
"eslint:test-server": "eslint --quiet --cache -c test-server/eslint.config.js ./test-server",
"lint": "pnpm typecheck && pnpm eslint",
"jest": "cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.unit.cjs",
"jest:e2e": "cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.e2e.cjs",
"jest:fed": "node ./jest.js --forceExit --config jest.config.fed.cjs",
"jest-and-coverage": "cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.unit.cjs",
"jest-and-coverage:e2e": "cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.e2e.cjs",
"jest": "cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.unit.ts",
"jest:e2e": "cross-env NODE_ENV=test node ./jest.js --forceExit --config jest.config.e2e.ts",
"jest:fed": "node ./jest.js --forceExit --config jest.config.fed.ts",
"jest-and-coverage": "cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.unit.ts",
"jest-and-coverage:e2e": "cross-env NODE_ENV=test node ./jest.js --coverage --forceExit --config jest.config.e2e.ts",
"jest-clear": "cross-env NODE_ENV=test node ./jest.js --clearCache",
"test": "pnpm jest",
"test:e2e": "pnpm build && pnpm build:test && pnpm jest:e2e",
"test:fed": "pnpm jest:fed",
"test-and-coverage": "pnpm jest-and-coverage",
"test-and-coverage:e2e": "pnpm build && pnpm build:test && pnpm jest-and-coverage:e2e",
"test:e2e": "pnpm run build && pnpm run build:test && pnpm run jest:e2e",
"test:fed": "pnpm run build && pnpm run build:test && pnpm run jest:fed",
"test-and-coverage": "pnpm run build && pnpm run build:test && pnpm run jest-and-coverage",
"test-and-coverage:e2e": "pnpm run build && pnpm run build:test && pnpm run jest-and-coverage:e2e",
"generate-api-json": "node ./scripts/generate_api_json.js"
},
"optionalDependencies": {
"@swc/core-android-arm64": "1.3.11",
"@swc/core-darwin-arm64": "1.11.24",
"@swc/core-darwin-x64": "1.11.24",
"@swc/core-darwin-arm64": "1.13.5",
"@swc/core-darwin-x64": "1.13.5",
"@swc/core-freebsd-x64": "1.3.11",
"@swc/core-linux-arm-gnueabihf": "1.11.24",
"@swc/core-linux-arm64-gnu": "1.11.24",
"@swc/core-linux-arm64-musl": "1.11.24",
"@swc/core-linux-x64-gnu": "1.11.24",
"@swc/core-linux-x64-musl": "1.11.24",
"@swc/core-win32-arm64-msvc": "1.11.24",
"@swc/core-win32-ia32-msvc": "1.11.24",
"@swc/core-win32-x64-msvc": "1.11.24",
"@swc/core-linux-arm-gnueabihf": "1.13.5",
"@swc/core-linux-arm64-gnu": "1.13.5",
"@swc/core-linux-arm64-musl": "1.13.5",
"@swc/core-linux-x64-gnu": "1.13.5",
"@swc/core-linux-x64-musl": "1.13.5",
"@swc/core-win32-arm64-msvc": "1.13.5",
"@swc/core-win32-ia32-msvc": "1.13.5",
"@swc/core-win32-x64-msvc": "1.13.5",
"bufferutil": "4.0.9",
"slacc-android-arm-eabi": "0.0.10",
"slacc-android-arm64": "0.0.10",
@ -68,86 +80,84 @@
"utf-8-validate": "6.0.5"
},
"dependencies": {
"@aws-sdk/client-s3": "3.797.0",
"@aws-sdk/lib-storage": "3.797.0",
"@discordapp/twemoji": "15.1.0",
"@aws-sdk/client-s3": "3.896.0",
"@aws-sdk/lib-storage": "3.896.0",
"@discordapp/twemoji": "16.0.1",
"@fastify/accepts": "5.0.2",
"@fastify/cookie": "11.0.2",
"@fastify/cors": "10.1.0",
"@fastify/cors": "11.1.0",
"@fastify/express": "4.0.2",
"@fastify/http-proxy": "10.0.2",
"@fastify/multipart": "9.0.3",
"@fastify/static": "8.1.1",
"@fastify/view": "10.0.2",
"@fastify/http-proxy": "11.3.0",
"@fastify/multipart": "9.2.1",
"@fastify/static": "8.2.0",
"@fastify/view": "11.1.1",
"@misskey-dev/sharp-read-bmp": "1.3.0",
"@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.2",
"@nestjs/common": "11.1.0",
"@nestjs/core": "11.1.0",
"@nestjs/testing": "11.1.0",
"@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.3",
"@nestjs/common": "11.1.6",
"@nestjs/core": "11.1.6",
"@nestjs/testing": "11.1.6",
"@peertube/http-signature": "1.7.0",
"@sentry/node": "8.55.0",
"@sentry/profiling-node": "8.55.0",
"@sentry/node": "10.15.0",
"@sentry/profiling-node": "10.15.0",
"@simplewebauthn/server": "12.0.0",
"@sinonjs/fake-timers": "11.3.1",
"@smithy/node-http-handler": "2.5.0",
"mfm-js": "npm:@transfem-org/sfm-js@0.24.8",
"@twemoji/parser": "15.1.1",
"@smithy/node-http-handler": "4.2.1",
"accepts": "1.3.8",
"ajv": "8.17.1",
"archiver": "7.0.1",
"argon2": "0.43.0",
"axios": "1.7.4",
"bcryptjs": "2.4.3",
"argon2": "0.44.0",
"axios": "1.12.2",
"bcryptjs": "3.0.2",
"blurhash": "2.0.5",
"bullmq": "5.51.1",
"bullmq": "5.58.7",
"cacheable-lookup": "7.0.0",
"canvas": "3.1.0",
"cbor": "9.0.2",
"chalk": "5.4.1",
"chalk-template": "1.1.0",
"cheerio": "1.0.0",
"cli-highlight": "npm:@transfem-org/cli-highlight@2.1.12",
"color-convert": "2.0.1",
"canvas": "3.2.0",
"cbor": "10.0.11",
"chalk": "5.6.2",
"chalk-template": "1.1.2",
"cheerio": "1.1.2",
"cli-highlight": "npm:@transfem-org/cli-highlight@2.1.13",
"color-convert": "3.1.2",
"content-disposition": "0.5.4",
"date-fns": "2.30.0",
"date-fns": "4.1.0",
"deep-email-validator": "0.1.21",
"dom-serializer": "2.0.0",
"domhandler": "5.0.3",
"domutils": "3.2.2",
"fastify": "5.3.2",
"fastify": "5.6.1",
"fastify-raw-body": "5.0.0",
"feed": "4.2.2",
"file-type": "19.6.0",
"feed": "5.1.0",
"file-type": "21.0.0",
"fluent-ffmpeg": "2.1.3",
"form-data": "4.0.2",
"glob": "11.0.0",
"got": "14.4.7",
"form-data": "4.0.4",
"glob": "11.0.3",
"got": "14.4.9",
"hpagent": "1.2.0",
"htmlescape": "1.1.1",
"htmlparser2": "9.1.0",
"ioredis": "5.6.1",
"htmlparser2": "10.0.0",
"ioredis": "5.8.0",
"ip-cidr": "4.0.2",
"ipaddr.js": "2.2.0",
"is-svg": "5.1.0",
"is-svg": "6.1.0",
"js-yaml": "4.1.0",
"json5": "2.2.3",
"jsonld": "8.3.3",
"juice": "11.0.1",
"megalodon": "workspace:*",
"meilisearch": "0.50.0",
"mime-types": "2.1.35",
"meilisearch": "0.53.0",
"mfm-js": "npm:@transfem-org/sfm-js@0.26.1",
"mime-types": "3.0.1",
"misskey-js": "workspace:*",
"misskey-reversi": "workspace:*",
"moment": "2.30.1",
"ms": "3.0.0-canary.1",
"nanoid": "5.1.5",
"ms": "3.0.0-canary.202508261828",
"nanoid": "5.1.6",
"nested-property": "4.0.0",
"node-fetch": "3.3.2",
"nodemailer": "6.10.1",
"nodemailer": "7.0.6",
"os-utils": "0.0.14",
"otpauth": "9.4.0",
"pg": "8.15.6",
"pkce-challenge": "4.1.0",
"otpauth": "9.4.1",
"pg": "8.16.3",
"pkce-challenge": "5.0.0",
"probe-image-size": "7.2.3",
"promise-limit": "2.7.0",
"proxy-addr": "2.0.7",
@ -155,58 +165,57 @@
"pug": "3.0.3",
"qrcode": "1.5.4",
"random-seed": "0.3.0",
"re2": "1.21.4",
"re2": "1.22.1",
"redis-info": "3.1.0",
"redis-lock": "0.1.4",
"redis-lock": "1.0.0",
"reflect-metadata": "0.2.2",
"rename": "1.0.4",
"sanitize-html": "2.16.0",
"secure-json-parse": "3.0.2",
"sharp": "0.34.1",
"semver": "7.7.1",
"sanitize-html": "2.17.0",
"secure-json-parse": "4.0.0",
"semver": "7.7.2",
"sharp": "0.34.4",
"slacc": "0.0.10",
"strict-event-emitter-types": "2.0.0",
"systeminformation": "5.25.11",
"systeminformation": "5.27.10",
"tinycolor2": "1.6.0",
"tmp": "0.2.3",
"tsc-alias": "1.8.15",
"tmp": "0.2.5",
"tsc-alias": "1.8.16",
"tsconfig-paths": "4.2.0",
"typeorm": "0.3.22",
"typescript": "5.8.3",
"ulid": "2.4.0",
"uuid": "11.1.0",
"typeorm": "0.3.27",
"ulid": "3.0.1",
"uuid": "13.0.0",
"vary": "1.1.2",
"web-push": "3.6.7",
"ws": "8.18.1",
"ws": "8.18.3",
"xev": "3.0.2"
},
"devDependencies": {
"@jest/globals": "29.7.0",
"@nestjs/platform-express": "11.1.0",
"@sentry/vue": "9.14.0",
"@jest/globals": "30.2.0",
"@nestjs/platform-express": "11.1.6",
"@sentry/vue": "10.15.0",
"@simplewebauthn/types": "12.0.0",
"@swc/cli": "0.7.3",
"@swc/core": "1.11.24",
"@swc/jest": "0.2.38",
"@swc/cli": "0.7.8",
"@swc/core": "1.13.5",
"@swc/jest": "0.2.39",
"@types/accepts": "1.3.7",
"@types/archiver": "6.0.3",
"@types/bcryptjs": "2.4.6",
"@types/bcryptjs": "3.0.0",
"@types/color-convert": "2.0.4",
"@types/content-disposition": "0.5.8",
"@types/content-disposition": "0.5.9",
"@types/fluent-ffmpeg": "2.1.27",
"@types/htmlescape": "1.1.3",
"@types/jest": "29.5.14",
"@types/jest": "30.0.0",
"@types/js-yaml": "4.0.9",
"@types/jsonld": "1.5.15",
"@types/jsrsasign": "10.5.15",
"@types/mime-types": "2.1.4",
"@types/ms": "0.7.34",
"@types/node": "22.15.2",
"@types/nodemailer": "6.4.17",
"@types/mime-types": "3.0.1",
"@types/ms": "2.1.0",
"@types/node": "22.18.1",
"@types/nodemailer": "7.0.1",
"@types/oauth": "0.9.6",
"@types/oauth2orize": "1.11.5",
"@types/oauth2orize-pkce": "0.1.2",
"@types/pg": "8.11.14",
"@types/pg": "8.15.5",
"@types/proxy-addr": "2.0.3",
"@types/psl": "1.1.3",
"@types/pug": "2.0.10",
@ -214,28 +223,32 @@
"@types/random-seed": "0.3.5",
"@types/redis-info": "3.0.3",
"@types/rename": "1.0.7",
"@types/sanitize-html": "2.15.0",
"@types/semver": "7.7.0",
"@types/sanitize-html": "2.16.0",
"@types/semver": "7.7.1",
"@types/simple-oauth2": "5.0.7",
"@types/sinonjs__fake-timers": "8.1.5",
"@types/supertest": "6.0.3",
"@types/tinycolor2": "1.4.6",
"@types/tmp": "0.2.6",
"@types/vary": "1.1.3",
"@types/web-push": "3.6.4",
"@types/ws": "8.18.1",
"@typescript-eslint/eslint-plugin": "8.31.0",
"@typescript-eslint/parser": "8.31.0",
"@typescript-eslint/eslint-plugin": "8.44.1",
"@typescript-eslint/parser": "8.44.1",
"aws-sdk-client-mock": "4.1.0",
"cross-env": "7.0.3",
"eslint-plugin-import": "2.31.0",
"execa": "9.5.2",
"cross-env": "10.0.0",
"eslint": "9.36.0",
"eslint-plugin-import": "2.32.0",
"execa": "9.6.0",
"fkill": "9.0.0",
"jest": "29.7.0",
"jest-mock": "29.7.0",
"jest": "30.2.0",
"jest-environment-node": "30.2.0",
"jest-mock": "30.2.0",
"nodemon": "3.1.10",
"pid-port": "1.0.2",
"pid-port": "2.0.0",
"simple-oauth2": "5.1.0",
"supertest": "7.1.0"
"supertest": "7.1.4",
"ts-jest": "29.4.5",
"ts-node": "10.9.2",
"typescript": "5.9.2"
}
}

12
packages/backend/scripts/check_connect.js
View file

@ -6,14 +6,18 @@
import Redis from 'ioredis';
import { loadConfig } from '../built/config.js';
import { createPostgresDataSource } from '../built/postgres.js';
import { LoggerService } from '../built/core/LoggerService.js';
import { NativeTimeService } from '../built/global/TimeService.js';
import { EnvService } from '../built/global/EnvService.js';
const config = loadConfig();
const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService());
const config = loadConfig(loggerService);
// createPostgresDataSource handles primaries and replicas automatically.
// usually, it only opens connections first use, so we force it using
// .initialize()
async function connectToPostgres(){
const source = createPostgresDataSource(config);
async function connectToPostgres() {
const source = createPostgresDataSource(config, loggerService);
await source.initialize();
await source.destroy();
}
@ -32,10 +36,8 @@ async function connectToRedis(redisOptions) {
try {
await redis.connect();
resolve();
} catch (e) {
reject(e);
} finally {
redis.disconnect(false);
}

12
packages/backend/scripts/dev.mjs
View file

@ -5,7 +5,7 @@
import { execa, execaNode } from 'execa';
/** @type {import('execa').ExecaChildProcess | undefined} */
/** @type {import('execa').ResultPromise | undefined} */
let backendProcess;
async function execBuildAssets() {
@ -13,7 +13,7 @@ async function execBuildAssets() {
cwd: '../../',
stdout: process.stdout,
stderr: process.stderr,
})
});
}
function execStart() {
@ -32,8 +32,8 @@ async function killProc() {
if (backendProcess) {
backendProcess.catch(() => {}); // backendProcess.kill()によって発生する例外を無視するためにcatch()を呼び出す
backendProcess.kill();
await new Promise(resolve => backendProcess.on('exit', resolve));
backendProcess = undefined;
await new Promise(resolve => backendProcess?.on('exit', resolve))
.finally(() => backendProcess = undefined);
}
}
@ -49,7 +49,7 @@ async function killProc() {
stdio: [process.stdin, process.stdout, process.stderr, 'ipc'],
serialization: "json",
})
.on('message', async (message) => {
.on('message', /** @param {{type: string}} message */ async (message) => {
if (message.type === 'exit') {
// かならずbuild->build-assetsの順番で呼び出したいので、
// 少々トリッキーだがnodemonからのexitイベントを利用してbuild-assets->startを行う。
@ -59,5 +59,5 @@ async function killProc() {
await execBuildAssets();
execStart();
}
})
});
})();

6
packages/backend/scripts/generate_api_json.js
View file

@ -5,6 +5,9 @@
import { execa } from 'execa';
import { writeFileSync, existsSync } from "node:fs";
import { LoggerService } from '../built/core/LoggerService.js';
import { NativeTimeService } from '../built/global/TimeService.js';
import { EnvService } from '../built/global/EnvService.js';
async function main() {
if (!process.argv.includes('--no-build')) {
@ -24,7 +27,8 @@ async function main() {
/** @type {import('../src/server/api/openapi/gen-spec.js')} */
const { genOpenapiSpec } = await import('../built/server/api/openapi/gen-spec.js');
const config = loadConfig();
const loggerService = new LoggerService(console, new NativeTimeService(), new EnvService());
const config = loadConfig(loggerService);
const spec = genOpenapiSpec(config, true);
writeFileSync('./built/api.json', JSON.stringify(spec), 'utf-8');

6
packages/backend/scripts/watch.mjs
View file

@ -8,20 +8,20 @@ import { execa } from 'execa';
(async () => {
// なぜかchokidarが動かない影響で、watchされない
/*
execa('tsc-alias', ['-w', '-p', 'tsconfig.json'], {
execa('tsc-alias', ['-w', '-p', 'tsconfig.backend.json'], {
stdout: process.stdout,
stderr: process.stderr,
});
*/
setInterval(() => {
execa('tsc-alias', ['-p', 'tsconfig.json'], {
execa('tsc-alias', ['-p', 'tsconfig.backend.json'], {
stdout: process.stdout,
stderr: process.stderr,
});
}, 3000);
execa('tsc', ['-w', '-p', 'tsconfig.json'], {
execa('tsc', ['-w', '-p', 'tsconfig.backend.json'], {
stdout: process.stdout,
stderr: process.stderr,
});

9
packages/backend/src/@types/redis-lock.d.ts vendored
View file

@ -4,9 +4,14 @@
*/
declare module 'redis-lock' {
import type Redis from 'ioredis';
export interface NodeRedis {
readonly v4: true;
set(key: string, value: string | number, opts?: { PX?: number, NX?: boolean }): Promise<'OK' | null>;
del(key: string): Promise<number>;
}
type Lock = (lockName: string, timeout?: number, taskToPerform?: () => Promise<void>) => void;
export type Unlock = () => Promise<void>;
export type Lock = (lockName: string, timeout?: number) => Promise<Unlock>;
function redisLock(client: Redis.Redis, retryDelay: number): Lock;
export = redisLock;

60
packages/backend/src/GlobalModule.ts
View file

@ -8,6 +8,14 @@ import * as Redis from 'ioredis';
import { DataSource } from 'typeorm';
import { MeiliSearch } from 'meilisearch';
import { MiMeta } from '@/models/Meta.js';
import { bindThis } from '@/decorators.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { TimeService, NativeTimeService } from '@/global/TimeService.js';
import { EnvService } from '@/global/EnvService.js';
import { CacheManagementService } from '@/global/CacheManagementService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import { DependencyService } from '@/global/DependencyService.js';
import { LoggerService } from '@/core/LoggerService.js';
import { DI } from './di-symbols.js';
import { Config, loadConfig } from './config.js';
import { createPostgresDataSource } from './postgres.js';
@ -19,21 +27,17 @@ import type { Provider, OnApplicationShutdown } from '@nestjs/common';
const $config: Provider = {
provide: DI.config,
useValue: loadConfig(),
useFactory: (loggerService: LoggerService) => loadConfig(loggerService),
inject: [LoggerService],
};
const $db: Provider = {
provide: DI.db,
useFactory: async (config) => {
try {
const db = createPostgresDataSource(config);
return await db.initialize();
} catch (e) {
console.error('failed to initialize database connection', e);
throw e;
}
useFactory: async (config: Config, loggerService: LoggerService) => {
const db = createPostgresDataSource(config, loggerService);
return await db.initialize();
},
inject: [DI.config],
inject: [DI.config, LoggerService],
};
const $meilisearch: Provider = {
@ -163,11 +167,22 @@ const $meta: Provider = {
inject: [DI.db, DI.redisForSub],
};
const $CacheManagementService: Provider[] = [CacheManagementService, { provide: 'CacheManagementService', useExisting: CacheManagementService }];
const $InternalEventService: Provider[] = [InternalEventService, { provide: 'InternalEventService', useExisting: InternalEventService }];
const $TimeService: Provider[] = [
{ provide: TimeService, useClass: NativeTimeService },
{ provide: 'TimeService', useExisting: TimeService },
];
const $EnvService: Provider[] = [EnvService, { provide: 'EnvService', useExisting: EnvService }];
const $LoggerService: Provider[] = [LoggerService, { provide: 'LoggerService', useExisting: LoggerService }];
const $Console: Provider[] = [{ provide: DI.console, useFactory: () => global.console }]; // useValue will break overrideProvider for some reason
const $DependencyService: Provider[] = [DependencyService, { provide: 'DependencyService', useExisting: DependencyService }];
@Global()
@Module({
imports: [RepositoryModule],
providers: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit],
exports: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit, RepositoryModule],
providers: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit, $CacheManagementService, $InternalEventService, $TimeService, $EnvService, $LoggerService, $Console, $DependencyService].flat(),
exports: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit, $CacheManagementService, $InternalEventService, $TimeService, $EnvService, $LoggerService, RepositoryModule, $Console, $DependencyService].flat(),
})
export class GlobalModule implements OnApplicationShutdown {
private readonly logger = new Logger('global');
@ -189,16 +204,25 @@ export class GlobalModule implements OnApplicationShutdown {
// And then disconnect from DB
this.logger.info('Disconnected from data sources...');
await this.db.destroy();
this.redisClient.disconnect();
this.redisForPub.disconnect();
this.redisForSub.disconnect();
this.redisForTimelines.disconnect();
this.redisForReactions.disconnect();
this.redisForRateLimit.disconnect();
this.safeDisconnect(this.redisClient);
this.safeDisconnect(this.redisForPub);
this.safeDisconnect(this.redisForSub);
this.safeDisconnect(this.redisForTimelines);
this.safeDisconnect(this.redisForReactions);
this.safeDisconnect(this.redisForRateLimit);
this.logger.info('Global module disposed.');
}
@bindThis
async onApplicationShutdown(signal: string): Promise<void> {
await this.dispose();
}
private safeDisconnect(redis: { disconnect(): void }): void {
try {
redis.disconnect();
} catch (err) {
this.logger.error(`Unhandled error disconnecting redis: ${renderInlineError(err)}`);
}
}
}

2
packages/backend/src/MainModule.ts
View file

@ -5,12 +5,10 @@
import { Module } from '@nestjs/common';
import { ServerModule } from '@/server/ServerModule.js';
import { GlobalModule } from '@/GlobalModule.js';
import { DaemonModule } from '@/daemons/DaemonModule.js';
@Module({
imports: [
GlobalModule,
ServerModule,
DaemonModule,
],

5
packages/backend/src/NestLogger.ts
View file

@ -4,10 +4,9 @@
*/
import { LoggerService } from '@nestjs/common';
import Logger from '@/logger.js';
import { coreLogger } from '@/boot/coreLogger.js';
const logger = new Logger('core', 'cyan');
const nestLogger = logger.createSubLogger('nest', 'green');
const nestLogger = coreLogger.createSubLogger('nest', 'green');
export class NestLogger implements LoggerService {
/**

6
packages/backend/src/boot/common.ts
View file

@ -12,7 +12,7 @@ import { QueueStatsService } from '@/daemons/QueueStatsService.js';
import { ServerStatsService } from '@/daemons/ServerStatsService.js';
import { ServerService } from '@/server/ServerService.js';
import { MainModule } from '@/MainModule.js';
import { envOption } from '@/env.js';
import { EnvService } from '@/global/EnvService.js';
import { ApLogCleanupService } from '@/daemons/ApLogCleanupService.js';
export async function server() {
@ -27,7 +27,9 @@ export async function server() {
if (process.env.NODE_ENV !== 'test') {
app.get(ChartManagementService).start();
}
if (!envOption.noDaemons) {
const envService = app.get(EnvService);
if (!envService.options.noDaemons) {
app.get(QueueStatsService).start();
app.get(ServerStatsService).start();
app.get(ApLogCleanupService).start();

14
packages/backend/src/boot/coreLogger.ts Normal file
View file

@ -0,0 +1,14 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { EnvService } from '@/global/EnvService.js';
import { LoggerService } from '@/core/LoggerService.js';
import { NativeTimeService } from '@/global/TimeService.js';
export const coreEnvService = new EnvService();
// eslint-disable-next-line no-restricted-globals
export const coreLoggerService = new LoggerService(console, new NativeTimeService(), coreEnvService);
export const coreLogger = coreLoggerService.getLogger('core', 'cyan');

53
packages/backend/src/boot/entry.ts
View file

@ -8,31 +8,28 @@
*/
import cluster from 'node:cluster';
import { EventEmitter } from 'node:events';
import { inspect } from 'node:util';
import chalk from 'chalk';
import Xev from 'xev';
import Logger from '@/logger.js';
import { envOption } from '../env.js';
import { coreLogger, coreEnvService, coreLoggerService } from '@/boot/coreLogger.js';
import { prepEnv } from '@/boot/prepEnv.js';
import { masterMain } from './master.js';
import { workerMain } from './worker.js';
import { readyRef } from './ready.js';
import 'reflect-metadata';
process.title = `Misskey (${cluster.isPrimary ? 'master' : 'worker'})`;
Error.stackTraceLimit = Infinity;
EventEmitter.defaultMaxListeners = 128;
prepEnv();
const logger = new Logger('core', 'cyan');
const clusterLogger = logger.createSubLogger('cluster', 'orange');
const ev = new Xev();
// We wrap this in a main function, that gets called,
// because not all platforms support top level await :/
async function main() {
const envOption = coreEnvService.options;
const clusterLogger = coreLogger.createSubLogger('cluster', 'orange');
const logger = coreLogger;
//#region Events
// Listen new workers
cluster.on('fork', worker => {
@ -52,36 +49,6 @@ async function main() {
cluster.fork();
});
// Display detail of unhandled promise rejection
if (!envOption.quiet) {
process.on('unhandledRejection', e => {
logger.error('Unhandled rejection:', inspect(e));
});
}
process.on('uncaughtException', (err) => {
// Workaround for https://github.com/node-fetch/node-fetch/issues/954
if (String(err).match(/^TypeError: .+ is an? url with embedded credentials.$/)) {
logger.debug('Suppressed node-fetch issue#954, but the current job may fail.');
return;
}
// Workaround for https://github.com/node-fetch/node-fetch/issues/1845
if (String(err) === 'TypeError: Cannot read properties of undefined (reading \'body\')') {
logger.debug('Suppressed node-fetch issue#1845, but the current job may fail.');
return;
}
// Throw all other errors to avoid inconsistent state.
// (per NodeJS docs, it's unsafe to suppress arbitrary errors in an uncaughtException handler.)
throw err;
});
// Display detail of uncaught exception
process.on('uncaughtExceptionMonitor', (err, origin) => {
logger.error(`Uncaught exception (${origin}):`, err);
});
// Dying away...
process.on('disconnect', () => {
logger.warn('IPC channel disconnected! The process may soon die.');
@ -97,18 +64,18 @@ async function main() {
if (!envOption.disableClustering) {
if (cluster.isPrimary) {
logger.info(`Start main process... pid: ${process.pid}`);
await masterMain();
await masterMain(coreLoggerService, coreEnvService);
ev.mount();
} else if (cluster.isWorker) {
logger.info(`Start worker process... pid: ${process.pid}`);
await workerMain();
await workerMain(coreLoggerService, coreEnvService);
} else {
throw new Error('Unknown process type');
}
} else {
// 非clusterの場合はMasterのみが起動するため、Workerの処理は行わない(cluster.isWorker === trueの状態でこのブロックに来ることはない)
logger.info(`Start main process... pid: ${process.pid}`);
await masterMain();
await masterMain(coreLoggerService, coreEnvService);
ev.mount();
}

86
packages/backend/src/boot/master.ts
View file

@ -13,11 +13,15 @@ import chalk from 'chalk';
import chalkTemplate from 'chalk-template';
import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';
import Logger from '@/logger.js';
import type Logger from '@/logger.js';
import { loadConfig } from '@/config.js';
import type { Config } from '@/config.js';
import type { LoggerService } from '@/core/LoggerService.js';
import type { EnvService } from '@/global/EnvService.js';
import type { EnvOption } from '@/env.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { showMachineInfo } from '@/misc/show-machine-info.js';
import { envOption } from '@/env.js';
import { coreLogger } from '@/boot/coreLogger.js';
import { jobQueue, server } from './common.js';
const _filename = fileURLToPath(import.meta.url);
@ -25,29 +29,26 @@ const _dirname = dirname(_filename);
const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json`, 'utf-8'));
const logger = new Logger('core', 'cyan');
const bootLogger = logger.createSubLogger('boot', 'magenta');
const themeColor = chalk.hex('#86b300');
function greet() {
function greet(logger: Logger, bootLogger: Logger, envOption: EnvOption) {
if (!envOption.quiet) {
//#region Misskey logo
console.log(themeColor(' _____ _ _ '));
console.log(themeColor('/ ___| | | | '));
console.log(themeColor('\\ `--.| |__ __ _ _ __| | _____ _ _ '));
console.log(themeColor(' `--. \\ \'_ \\ / _` | \'__| |/ / _ \\ | | |'));
console.log(themeColor('/\\__/ / | | | (_| | | | < __/ |_| |'));
console.log(themeColor('\\____/|_| |_|\\__,_|_| |_|\\_\\___|\\__, |'));
console.log(themeColor(' __/ |'));
console.log(themeColor(' |___/ '));
logger.info(themeColor(' _____ _ _ '));
logger.info(themeColor('/ ___| | | | '));
logger.info(themeColor('\\ `--.| |__ __ _ _ __| | _____ _ _ '));
logger.info(themeColor(' `--. \\ \'_ \\ / _` | \'__| |/ / _ \\ | | |'));
logger.info(themeColor('/\\__/ / | | | (_| | | | < __/ |_| |'));
logger.info(themeColor('\\____/|_| |_|\\__,_|_| |_|\\_\\___|\\__, |'));
logger.info(themeColor(' __/ |'));
logger.info(themeColor(' |___/ '));
//#endregion
console.log(' Sharkey is an open-source decentralized microblogging platform.');
console.log(chalk.rgb(255, 136, 0)(' If you like Sharkey, please donate to support development. https://opencollective.com/sharkey'));
logger.info(' Sharkey is an open-source decentralized microblogging platform.');
logger.info(chalk.rgb(255, 136, 0)(' If you like Sharkey, please donate to support development. https://opencollective.com/sharkey'));
console.log('');
console.log(chalkTemplate`--- ${os.hostname()} {gray (PID: ${process.pid.toString()})} ---`);
logger.info('');
logger.info(chalkTemplate`--- ${os.hostname()} {gray (PID: ${process.pid.toString()})} ---`);
}
bootLogger.info('Welcome to Sharkey!');
@ -57,20 +58,23 @@ function greet() {
/**
* Init master process
*/
export async function masterMain() {
export async function masterMain(loggerService: LoggerService, envService: EnvService) {
let config!: Config;
const bootLogger = coreLogger.createSubLogger('boot', 'magenta');
const envOption = envService.options;
// initialize app
try {
greet();
showEnvironment();
greet(coreLogger, bootLogger, envOption);
showEnvironment(bootLogger);
await showMachineInfo(bootLogger);
showNodejsVersion();
config = loadConfigBoot();
showNodejsVersion(bootLogger);
config = loadConfig(loggerService);
//await connectDb();
if (config.pidFile) fs.writeFileSync(config.pidFile, process.pid.toString());
} catch (e) {
bootLogger.error('Fatal error occurred during initialization', null, true);
bootLogger.error(`Fatal error occurred during initialization: ${renderInlineError(e)}`, { e }, true);
process.exit(1);
}
@ -125,7 +129,7 @@ export async function masterMain() {
process.exit(1);
}
await spawnWorkers(config.clusterLimit);
await spawnWorkers(bootLogger, config.clusterLimit);
} else {
// clusterモジュール無効時
@ -147,7 +151,7 @@ export async function masterMain() {
}
}
function showEnvironment(): void {
function showEnvironment(bootLogger: Logger): void {
const env = process.env.NODE_ENV;
const logger = bootLogger.createSubLogger('env');
logger.info(typeof env === 'undefined' ? 'NODE_ENV is not set' : `NODE_ENV: ${env}`);
@ -158,34 +162,12 @@ function showEnvironment(): void {
}
}
function showNodejsVersion(): void {
function showNodejsVersion(bootLogger: Logger): void {
const nodejsLogger = bootLogger.createSubLogger('nodejs');
nodejsLogger.info(`Version ${process.version} detected.`);
}
function loadConfigBoot(): Config {
const configLogger = bootLogger.createSubLogger('config');
let config;
try {
config = loadConfig();
} catch (exception) {
if (typeof exception === 'string') {
configLogger.error('Exception loading config:', exception);
process.exit(1);
} else if ((exception as any).code === 'ENOENT') {
configLogger.error('Configuration file not found', null, true);
process.exit(1);
}
throw exception;
}
configLogger.info('Loaded');
return config;
}
/*
async function connectDb(): Promise<void> {
const dbLogger = bootLogger.createSubLogger('db');
@ -204,17 +186,17 @@ async function connectDb(): Promise<void> {
}
*/
async function spawnWorkers(limit = 1) {
async function spawnWorkers(bootLogger: Logger, limit = 1) {
const cpuCount = os.cpus().length;
// in some weird environments, node can't count the CPUs; we trust the config in those cases
const workers = cpuCount === 0 ? limit : Math.min(limit, cpuCount);
bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`);
await Promise.all([...Array(workers)].map(spawnWorker));
await Promise.all([...Array(workers)].map(() => spawnWorker(bootLogger)));
bootLogger.info('All workers started');
}
function spawnWorker(): Promise<void> {
function spawnWorker(bootLogger: Logger): Promise<void> {
return new Promise(res => {
const worker = cluster.fork();
worker.on('message', message => {

56
packages/backend/src/boot/prepEnv.ts Normal file
View file

@ -0,0 +1,56 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { EventEmitter } from 'node:events';
import { inspect } from 'node:util';
import { coreLogger } from '@/boot/coreLogger.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
// Polyfill reflection metadata *without* loading dependencies that may corrupt native types.
// https://github.com/microsoft/reflect-metadata?tab=readme-ov-file#es-modules-in-nodejsbrowser-typescriptbabel-bundlers
import 'reflect-metadata/lite';
/**
* Configures Node.JS global runtime options for values appropriate for Sharkey.
*/
export function prepEnv() {
// Increase maximum stack trace length.
// This helps diagnose infinite recursion bugs.
Error.stackTraceLimit = Infinity;
// Avoid warnings like "11 message listeners added to [Commander]. MaxListeners is 10."
// This is expected due to use of NestJS lifecycle hooks.
EventEmitter.defaultMaxListeners = 128;
// Workaround certain 3rd-party bugs
process.on('uncaughtException', (err) => {
// Workaround for https://github.com/node-fetch/node-fetch/issues/954
if (String(err).match(/^TypeError: .+ is an? url with embedded credentials.$/)) {
coreLogger.debug('Suppressed node-fetch issue#954, but the current job may fail.');
return;
}
// Workaround for https://github.com/node-fetch/node-fetch/issues/1845
if (String(err) === 'TypeError: Cannot read properties of undefined (reading \'body\')') {
coreLogger.debug('Suppressed node-fetch issue#1845, but the current job may fail.');
return;
}
// Throw all other errors to avoid inconsistent state.
// (per NodeJS docs, it's unsafe to suppress arbitrary errors in an uncaughtException handler.)
coreLogger.error(`Uncaught exception: ${renderInlineError(err)}`, {
error: inspect(err),
});
throw err;
});
// Log uncaught promise rejections
process.on('unhandledRejection', (error, promise) => {
coreLogger.error(`Unhandled rejection: ${renderInlineError(error)}`, {
error: inspect(error),
promise: inspect(promise),
});
});
}

19
packages/backend/src/boot/worker.ts
View file

@ -4,14 +4,16 @@
*/
import cluster from 'node:cluster';
import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';
import { envOption } from '@/env.js';
import { loadConfig } from '@/config.js';
import { jobQueue, server } from './common.js';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import * as fs from 'node:fs';
import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';
import { loadConfig } from '@/config.js';
import type { LoggerService } from '@/core/LoggerService.js';
import type { EnvService } from '@/global/EnvService.js';
import { jobQueue, server } from './common.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json`, 'utf-8'));
@ -19,8 +21,9 @@ const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json
/**
* Init worker process
*/
export async function workerMain() {
const config = loadConfig();
export async function workerMain(loggerService: LoggerService, envService: EnvService) {
const config = loadConfig(loggerService);
const envOption = envService.options;
if (config.sentryForBackend) {
Sentry.init({
@ -37,7 +40,7 @@ export async function workerMain() {
maxBreadcrumbs: 0,
// Set release version
release: "Sharkey@" + (meta.gitVersion ?? meta.version),
release: 'Sharkey@' + (meta.gitVersion ?? meta.version),
...config.sentryForBackend.options,
});

20
packages/backend/src/config.ts
View file

@ -14,6 +14,7 @@ import type * as Sentry from '@sentry/node';
import type * as SentryVue from '@sentry/vue';
import type { RedisOptions } from 'ioredis';
import type { IPv4, IPv6 } from 'ipaddr.js';
import type { LoggerService } from '@/core/LoggerService.js';
type RedisOptionsSource = Partial<RedisOptions> & {
host?: string;
@ -144,7 +145,6 @@ type Source = {
disableQueryTruncation?: boolean,
enableQueryParamLogging?: boolean,
};
verbose?: boolean;
}
activityLogging?: {
@ -160,8 +160,6 @@ type Source = {
}
};
const configLogger = new Logger('config');
export type PrivateNetworkSource = string | { network?: string, ports?: number[] };
export type PrivateNetwork = {
@ -180,10 +178,10 @@ export type PrivateNetwork = {
export type CIDR = [ip: IPv4 | IPv6, prefixLength: number];
export function parsePrivateNetworks(patterns: PrivateNetworkSource[]): PrivateNetwork[];
export function parsePrivateNetworks(patterns: undefined): undefined;
export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined): PrivateNetwork[] | undefined;
export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined): PrivateNetwork[] | undefined {
export function parsePrivateNetworks(patterns: PrivateNetworkSource[], configLogger: Logger): PrivateNetwork[];
export function parsePrivateNetworks(patterns: undefined, configLogger: Logger): undefined;
export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined, configLogger: Logger): PrivateNetwork[] | undefined;
export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefined, configLogger: Logger): PrivateNetwork[] | undefined {
if (!patterns) return undefined;
return patterns
.map(e => {
@ -290,7 +288,6 @@ export type Config = {
disableQueryTruncation?: boolean,
enableQueryParamLogging?: boolean,
};
verbose?: boolean;
}
version: string;
@ -370,7 +367,9 @@ const path = process.env.MISSKEY_CONFIG_YML
? resolve(dir, 'test.yml')
: resolve(dir, 'default.yml');
export function loadConfig(): Config {
export function loadConfig(loggerService: LoggerService): Config {
const configLogger = loggerService.getLogger('config');
const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8'));
const frontendManifestExists = fs.existsSync(_dirname + '/../../../built/_frontend_vite_/manifest.json');
@ -458,7 +457,7 @@ export function loadConfig(): Config {
proxy: config.proxy,
proxySmtp: config.proxySmtp,
proxyBypassHosts: config.proxyBypassHosts,
allowedPrivateNetworks: parsePrivateNetworks(config.allowedPrivateNetworks),
allowedPrivateNetworks: parsePrivateNetworks(config.allowedPrivateNetworks, configLogger),
disallowExternalApRedirect: config.disallowExternalApRedirect ?? false,
maxFileSize: config.maxFileSize ?? 262144000,
maxNoteLength: config.maxNoteLength ?? 3000,
@ -671,7 +670,6 @@ function applyEnvOverrides(config: Source) {
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
_apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword', 'disallowExternalApRedirect']]);
_apply_top(['logging', 'sql', ['disableQueryTruncation', 'enableQueryParamLogging']]);
_apply_top(['logging', ['verbose']]);
_apply_top(['activityLogging', ['enabled', 'preSave', 'maxAge']]);
_apply_top(['customHtml', ['head']]);
}

4
packages/backend/src/core/AbuseReportNotificationService.ts
View file

@ -23,6 +23,7 @@ import { RecipientMethod } from '@/models/AbuseReportNotificationRecipient.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { SystemWebhookService } from '@/core/SystemWebhookService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { TimeService } from '@/global/TimeService.js';
import { IdService } from './IdService.js';
@Injectable()
@ -44,6 +45,7 @@ export class AbuseReportNotificationService implements OnApplicationShutdown {
private moderationLogService: ModerationLogService,
private globalEventService: GlobalEventService,
private userEntityService: UserEntityService,
private readonly timeService: TimeService,
) {
this.redisForSub.on('message', this.onMessage);
}
@ -326,7 +328,7 @@ export class AbuseReportNotificationService implements OnApplicationShutdown {
await this.abuseReportNotificationRecipientRepository.update(params.id, {
isActive: params.isActive,
updatedAt: new Date(),
updatedAt: this.timeService.date,
name: params.name,
method: params.method,
userId: params.userId,

74
packages/backend/src/core/AccountMoveService.ts
View file

@ -27,6 +27,8 @@ import { SystemAccountService } from '@/core/SystemAccountService.js';
import { RoleService } from '@/core/RoleService.js';
import { AntennaService } from '@/core/AntennaService.js';
import { CacheService } from '@/core/CacheService.js';
import { UserListService } from '@/core/UserListService.js';
import { TimeService } from '@/global/TimeService.js';
@Injectable()
export class AccountMoveService {
@ -70,6 +72,8 @@ export class AccountMoveService {
private roleService: RoleService,
private antennaService: AntennaService,
private readonly cacheService: CacheService,
private readonly userListService: UserListService,
private readonly timeService: TimeService,
) {
}
@ -87,7 +91,7 @@ export class AccountMoveService {
const update = {} as Partial<MiLocalUser>;
update.alsoKnownAs = src.alsoKnownAs?.includes(dstUri) ? src.alsoKnownAs : src.alsoKnownAs?.concat([dstUri]) ?? [dstUri];
update.movedToUri = dstUri;
update.movedAt = new Date();
update.movedAt = this.timeService.date;
await this.usersRepository.update(src.id, update);
Object.assign(src, update);
@ -179,7 +183,7 @@ export class AccountMoveService {
// Insert new mutings with the same values except mutee
const oldMutings = await this.mutingsRepository.findBy([
{ muteeId: src.id, expiresAt: IsNull() },
{ muteeId: src.id, expiresAt: MoreThan(new Date()) },
{ muteeId: src.id, expiresAt: MoreThan(this.timeService.date) },
]);
if (oldMutings.length === 0) return;
@ -263,45 +267,35 @@ export class AccountMoveService {
@bindThis
public async updateLists(src: ThinUser, dst: MiUser): Promise<void> {
// Return if there is no list to be updated.
const oldMemberships = await this.userListMembershipsRepository.find({
where: {
userId: src.id,
},
});
if (oldMemberships.length === 0) return;
const [srcMemberships, dstMemberships] = await Promise.all([
this.cacheService.userListMembershipsCache.fetch(src.id),
this.cacheService.userListMembershipsCache.fetch(dst.id),
]);
if (srcMemberships.size === 0) return;
const existingUserListIds = await this.userListMembershipsRepository.find({
where: {
userId: dst.id,
},
}).then(memberships => memberships.map(membership => membership.userListId));
const newMemberships = srcMemberships.values()
.filter(srcMembership => !dstMemberships.has(srcMembership.userListId))
.map(srcMembership => ({
userListId: srcMembership.userListId,
withReplies: srcMembership.withReplies,
}))
.toArray();
const updatedMemberships = srcMemberships.values()
.filter(srcMembership => {
const dstMembership = dstMemberships.get(srcMembership.userListId);
return dstMembership != null && dstMembership.withReplies !== srcMembership.withReplies;
})
.map(srcMembership => ({
userListId: srcMembership.userListId,
withReplies: srcMembership.withReplies,
}))
.toArray();
const newMemberships: Map<string, { userId: string; userListId: string; userListUserId: string; }> = new Map();
// 重複しないようにIDを生成
const genId = (): string => {
let id: string;
do {
id = this.idService.gen();
} while (newMemberships.has(id));
return id;
};
for (const membership of oldMemberships) {
if (existingUserListIds.includes(membership.userListId)) continue; // skip if dst exists in this user's list
newMemberships.set(genId(), {
userId: dst.id,
userListId: membership.userListId,
userListUserId: membership.userListUserId,
});
if (newMemberships.length > 0) {
await this.userListService.bulkAddMember(dst, newMemberships);
}
const arrayToInsert = Array.from(newMemberships.entries()).map(entry => ({ ...entry[1], id: entry[0] }));
await this.userListMembershipsRepository.insert(arrayToInsert);
// Have the proxy account follow the new account in the same way as UserListService.push
if (this.userEntityService.isRemoteUser(dst)) {
const proxy = await this.systemAccountService.fetch('proxy');
this.queueService.createFollowJob([{ from: { id: proxy.id }, to: { id: dst.id } }]);
if (updatedMemberships.length > 0) {
await this.userListService.bulkUpdateMembership(dst, updatedMemberships);
}
}
@ -356,7 +350,7 @@ export class AccountMoveService {
let resultUser: MiLocalUser | MiRemoteUser | null = null;
if (this.userEntityService.isRemoteUser(dst)) {
if (Date.now() - (dst.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) {
if (this.timeService.now - (dst.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) {
await this.apPersonService.updatePerson(dst.uri);
}
dst = await this.apPersonService.fetchPerson(dst.uri) ?? dst;
@ -372,7 +366,7 @@ export class AccountMoveService {
if (!src) continue; // oldAccountを探してもこのサーバーに存在しない場合はフォロー関係もないということなのでスルー
if (this.userEntityService.isRemoteUser(dst)) {
if (Date.now() - (src.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) {
if (this.timeService.now - (src.lastFetchedAt?.getTime() ?? 0) > 10 * 1000) {
await this.apPersonService.updatePerson(srcUri);
}

4
packages/backend/src/core/AchievementService.ts
View file

@ -9,6 +9,7 @@ import type { MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { NotificationService } from '@/core/NotificationService.js';
import { TimeService } from '@/global/TimeService.js';
import { ACHIEVEMENT_TYPES } from '@/models/UserProfile.js';
@Injectable()
@ -18,6 +19,7 @@ export class AchievementService {
private userProfilesRepository: UserProfilesRepository,
private notificationService: NotificationService,
private readonly timeService: TimeService,
) {
}
@ -28,7 +30,7 @@ export class AchievementService {
): Promise<void> {
if (!ACHIEVEMENT_TYPES.includes(type)) return;
const date = Date.now();
const date = this.timeService.now;
const profile = await this.userProfilesRepository.findOneByOrFail({ userId: userId });

4
packages/backend/src/core/AnnouncementService.ts
View file

@ -17,6 +17,7 @@ import { ModerationLogService } from '@/core/ModerationLogService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import type { Config } from '@/config.js';
import { RoleService } from '@/core/RoleService.js';
import { TimeService } from '@/global/TimeService.js';
@Injectable()
export class AnnouncementService {
@ -38,6 +39,7 @@ export class AnnouncementService {
private moderationLogService: ModerationLogService,
private announcementEntityService: AnnouncementEntityService,
private roleService: RoleService,
private readonly timeService: TimeService,
) {
}
@ -143,7 +145,7 @@ export class AnnouncementService {
}
await this.announcementsRepository.update(announcement.id, {
updatedAt: new Date(),
updatedAt: this.timeService.date,
title: values.title,
text: values.text,
/* eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- 空の文字列の場合、nullを渡すようにするため */

8
packages/backend/src/core/AntennaService.ts
View file

@ -139,12 +139,8 @@ export class AntennaService implements OnApplicationShutdown {
// TODO
} else if (antenna.src === 'list') {
if (antenna.userListId == null) return false;
const exists = await this.userListMembershipsRepository.exists({
where: {
userListId: antenna.userListId,
userId: note.userId,
},
});
const memberships = await this.cacheService.userListMembershipsCache.fetch(note.userId);
const exists = memberships.has(antenna.userListId);
if (!exists) return false;
} else if (antenna.src === 'users') {
const accts = antenna.users.map(x => {

8
packages/backend/src/core/ApLogService.ts
View file

@ -12,6 +12,7 @@ import type { ApContextsRepository, ApFetchLogsRepository, ApInboxLogsRepository
import type { Config } from '@/config.js';
import { JsonValue } from '@/misc/json-value.js';
import { UtilityService } from '@/core/UtilityService.js';
import { TimeService } from '@/global/TimeService.js';
import { IdService } from '@/core/IdService.js';
import { IActivity, IObject } from './activitypub/type.js';
@ -32,6 +33,7 @@ export class ApLogService {
private readonly utilityService: UtilityService,
private readonly idService: IdService,
private readonly timeService: TimeService,
) {}
/**
@ -46,7 +48,7 @@ export class ApLogService {
const log = new SkApInboxLog({
id: this.idService.gen(),
at: new Date(),
at: this.timeService.date,
verified: false,
accepted: false,
host,
@ -85,7 +87,7 @@ export class ApLogService {
}): Promise<SkApFetchLog> {
const log = new SkApFetchLog({
id: this.idService.gen(),
at: new Date(),
at: this.timeService.date,
accepted: false,
...data,
});
@ -163,7 +165,7 @@ export class ApLogService {
*/
public async deleteExpiredLogs(): Promise<number> {
// This is the date in UTC of the oldest log to KEEP
const oldestAllowed = new Date(Date.now() - this.config.activityLogging.maxAge);
const oldestAllowed = new Date(this.timeService.now - this.config.activityLogging.maxAge);
// Delete all logs older than the threshold.
const inboxDeleted = await this.deleteExpiredInboxLogs(oldestAllowed);

34
packages/backend/src/core/AppLockService.ts
View file

@ -5,7 +5,7 @@
import { promisify } from 'node:util';
import { Inject, Injectable } from '@nestjs/common';
import redisLock from 'redis-lock';
import redisLock, { Unlock, NodeRedis } from 'redis-lock';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
@ -17,13 +17,13 @@ const retryDelay = 100;
@Injectable()
export class AppLockService {
private lock: (key: string, timeout?: number, _?: (() => Promise<void>) | undefined) => Promise<() => void>;
private lock: (key: string, timeout?: number) => Promise<Unlock>;
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
) {
this.lock = promisify(redisLock(this.redisClient, retryDelay));
this.lock = redisLock(adaptRedis(this.redisClient), retryDelay);
}
/**
@ -33,12 +33,36 @@ export class AppLockService {
* @returns Unlock function
*/
@bindThis
public getApLock(uri: string, timeout = 30 * 1000): Promise<() => void> {
public getApLock(uri: string, timeout = 30 * 1000): Promise<Unlock> {
return this.lock(`ap-object:${uri}`, timeout);
}
@bindThis
public getChartInsertLock(lockKey: string, timeout = 30 * 1000): Promise<() => void> {
public getChartInsertLock(lockKey: string, timeout = 30 * 1000): Promise<Unlock> {
return this.lock(`chart-insert:${lockKey}`, timeout);
}
}
/**
* Adapts an ioredis instance into something close enough to NodeRedis that it works with redis-lock.
*/
function adaptRedis(ioredis: Redis.Redis): NodeRedis {
return {
v4: true,
async set(key: string, value: string | number, opts?: { PX?: number, NX?: boolean }) {
if (opts) {
if (opts.PX != null && opts.NX) {
return ioredis.set(key, value, 'PX', opts.PX, 'NX');
} else if (opts.PX != null) {
return ioredis.set(key, value, 'PX', opts.PX);
} else if (opts.NX) {
return ioredis.set(key, value, 'NX');
}
}
return ioredis.set(key, value);
},
async del(key: string) {
return await ioredis.del(key);
},
};
}

47
packages/backend/src/core/AvatarDecorationService.ts
View file

@ -13,10 +13,13 @@ import { bindThis } from '@/decorators.js';
import { MemorySingleCache } from '@/misc/cache.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { CacheManagementService, type ManagedMemorySingleCache } from '@/global/CacheManagementService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import { TimeService } from '@/global/TimeService.js';
@Injectable()
export class AvatarDecorationService implements OnApplicationShutdown {
public cache: MemorySingleCache<MiAvatarDecoration[]>;
public cache: ManagedMemorySingleCache<MiAvatarDecoration[]>;
constructor(
@Inject(DI.redisForSub)
@ -28,29 +31,21 @@ export class AvatarDecorationService implements OnApplicationShutdown {
private idService: IdService,
private moderationLogService: ModerationLogService,
private globalEventService: GlobalEventService,
) {
this.cache = new MemorySingleCache<MiAvatarDecoration[]>(1000 * 60 * 30); // 30s
private readonly internalEventService: InternalEventService,
private readonly timeService: TimeService,
this.redisForSub.on('message', this.onMessage);
cacheManagementService: CacheManagementService,
) {
this.cache = cacheManagementService.createMemorySingleCache<MiAvatarDecoration[]>('avatarDecorations', 1000 * 60 * 30); // 30s
this.internalEventService.on('avatarDecorationCreated', this.onAvatarEvent);
this.internalEventService.on('avatarDecorationUpdated', this.onAvatarEvent);
this.internalEventService.on('avatarDecorationDeleted', this.onAvatarEvent);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
switch (type) {
case 'avatarDecorationCreated':
case 'avatarDecorationUpdated':
case 'avatarDecorationDeleted': {
this.cache.delete();
break;
}
default:
break;
}
}
private onAvatarEvent(): void {
this.cache.delete();
}
@bindThis
@ -60,7 +55,7 @@ export class AvatarDecorationService implements OnApplicationShutdown {
...options,
});
this.globalEventService.publishInternalEvent('avatarDecorationCreated', created);
await this.internalEventService.emit('avatarDecorationCreated', created);
if (moderator) {
this.moderationLogService.log(moderator, 'createAvatarDecoration', {
@ -76,14 +71,14 @@ export class AvatarDecorationService implements OnApplicationShutdown {
public async update(id: MiAvatarDecoration['id'], params: Partial<MiAvatarDecoration>, moderator?: MiUser): Promise<void> {
const avatarDecoration = await this.avatarDecorationsRepository.findOneByOrFail({ id });
const date = new Date();
const date = this.timeService.date;
await this.avatarDecorationsRepository.update(avatarDecoration.id, {
updatedAt: date,
...params,
});
const updated = await this.avatarDecorationsRepository.findOneByOrFail({ id: avatarDecoration.id });
this.globalEventService.publishInternalEvent('avatarDecorationUpdated', updated);
await this.internalEventService.emit('avatarDecorationUpdated', updated);
if (moderator) {
this.moderationLogService.log(moderator, 'updateAvatarDecoration', {
@ -99,7 +94,7 @@ export class AvatarDecorationService implements OnApplicationShutdown {
const avatarDecoration = await this.avatarDecorationsRepository.findOneByOrFail({ id });
await this.avatarDecorationsRepository.delete({ id: avatarDecoration.id });
this.globalEventService.publishInternalEvent('avatarDecorationDeleted', avatarDecoration);
await this.internalEventService.emit('avatarDecorationDeleted', avatarDecoration);
if (moderator) {
this.moderationLogService.log(moderator, 'deleteAvatarDecoration', {
@ -119,7 +114,9 @@ export class AvatarDecorationService implements OnApplicationShutdown {
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.internalEventService.off('avatarDecorationCreated', this.onAvatarEvent);
this.internalEventService.off('avatarDecorationUpdated', this.onAvatarEvent);
this.internalEventService.off('avatarDecorationDeleted', this.onAvatarEvent);
}
@bindThis

34
packages/backend/src/core/BunnyService.ts
View file

@ -7,54 +7,58 @@ import * as https from 'node:https';
import * as fs from 'node:fs';
import { Readable } from 'node:stream';
import { finished } from 'node:stream/promises';
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import type { MiMeta } from '@/models/Meta.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import Logger from '@/logger.js';
import { DI } from '@/di-symbols.js';
@Injectable()
export class BunnyService {
private bunnyCdnLogger: Logger;
private readonly bunnyCdnLogger: Logger;
constructor(
@Inject(DI.meta)
private readonly meta: MiMeta,
private httpRequestService: HttpRequestService,
) {
this.bunnyCdnLogger = new Logger('bunnycdn', 'blue');
}
@bindThis
public getBunnyInfo(meta: MiMeta) {
if (!meta.objectStorageEndpoint || !meta.objectStorageBucket || !meta.objectStorageSecretKey) {
public getBunnyInfo() {
if (!this.meta.objectStorageEndpoint || !this.meta.objectStorageBucket || !this.meta.objectStorageSecretKey) {
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf90', 'Failed to use BunnyCDN, One of the required fields is missing.');
}
return {
endpoint: meta.objectStorageEndpoint,
endpoint: this.meta.objectStorageEndpoint,
/*
The way S3 works is that the Secret Key is essentially the password for the API but Bunny calls their password AccessKey so we call it accessKey here.
Bunny also doesn't specify a username/s3 access key when doing HTTP API requests so we end up not using our Access Key field from the form.
*/
accessKey: meta.objectStorageSecretKey,
zone: meta.objectStorageBucket,
fullUrl: `https://${meta.objectStorageEndpoint}/${meta.objectStorageBucket}`,
accessKey: this.meta.objectStorageSecretKey,
zone: this.meta.objectStorageBucket,
fullUrl: `https://${this.meta.objectStorageEndpoint}/${this.meta.objectStorageBucket}`,
};
}
@bindThis
public usingBunnyCDN(meta: MiMeta) {
return meta.objectStorageEndpoint && meta.objectStorageEndpoint.endsWith('bunnycdn.com');
public usingBunnyCDN() {
return this.meta.objectStorageEndpoint && this.meta.objectStorageEndpoint.endsWith('bunnycdn.com');
}
@bindThis
public async upload(meta: MiMeta, path: string, input: fs.ReadStream | Buffer) {
const client = this.getBunnyInfo(meta);
public async upload(path: string, input: fs.ReadStream | Buffer) {
const client = this.getBunnyInfo();
// Required to convert the buffer from webpublic and thumbnail to a ReadableStream for PUT
const data = Buffer.isBuffer(input) ? Readable.from(input) : input;
const agent = this.httpRequestService.getAgentByUrl(new URL(`${client.fullUrl}/${path}`), !meta.objectStorageUseProxy, true);
const agent = this.httpRequestService.getAgentByUrl(new URL(`${client.fullUrl}/${path}`), !this.meta.objectStorageUseProxy, true);
// Seperation of path and host/domain is required here
const options = {
@ -94,8 +98,8 @@ export class BunnyService {
}
@bindThis
public delete(meta: MiMeta, file: string) {
const client = this.getBunnyInfo(meta);
public delete(file: string) {
const client = this.getBunnyInfo();
return this.httpRequestService.send(`${client.fullUrl}/${file}`, { method: 'DELETE', headers: { AccessKey: client.accessKey } });
}
}

1018
packages/backend/src/core/CacheService.ts
View file

File diff suppressed because it is too large Load diff

12
packages/backend/src/core/CaptchaService.ts
View file

@ -10,23 +10,13 @@ import { MetaService } from '@/core/MetaService.js';
import { MiMeta } from '@/models/Meta.js';
import Logger from '@/logger.js';
import { LoggerService } from '@/core/LoggerService.js';
import { CaptchaError } from '@/misc/captcha-error.js';
import { CaptchaError, captchaErrorCodes } from '@/misc/captcha-error.js';
export { CaptchaError } from '@/misc/captcha-error.js';
export const supportedCaptchaProviders = ['none', 'hcaptcha', 'mcaptcha', 'recaptcha', 'turnstile', 'fc', 'testcaptcha'] as const;
export type CaptchaProvider = typeof supportedCaptchaProviders[number];
export const captchaErrorCodes = {
invalidProvider: Symbol('invalidProvider'),
invalidParameters: Symbol('invalidParameters'),
noResponseProvided: Symbol('noResponseProvided'),
requestFailed: Symbol('requestFailed'),
verificationFailed: Symbol('verificationFailed'),
unknown: Symbol('unknown'),
} as const;
export type CaptchaErrorCode = typeof captchaErrorCodes[keyof typeof captchaErrorCodes];
export type CaptchaSetting = {
provider: CaptchaProvider;
hcaptcha: {

57
packages/backend/src/core/ChannelFollowingService.ts
View file

@ -9,16 +9,13 @@ import { DI } from '@/di-symbols.js';
import type { ChannelFollowingsRepository } from '@/models/_.js';
import { MiChannel } from '@/models/_.js';
import { IdService } from '@/core/IdService.js';
import { GlobalEvents, GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { MiLocalUser } from '@/models/User.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { InternalEventService } from './InternalEventService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
@Injectable()
export class ChannelFollowingService implements OnModuleInit {
public userFollowingChannelsCache: QuantumKVCache<Set<string>>;
export class ChannelFollowingService {
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@ -29,21 +26,7 @@ export class ChannelFollowingService implements OnModuleInit {
private idService: IdService,
private globalEventService: GlobalEventService,
private readonly internalEventService: InternalEventService,
) {
this.userFollowingChannelsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userFollowingChannels', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: (key) => this.channelFollowingsRepository.find({
where: { followerId: key },
select: ['followeeId'],
}).then(xs => new Set(xs.map(x => x.followeeId))),
});
this.internalEventService.on('followChannel', this.onMessage);
this.internalEventService.on('unfollowChannel', this.onMessage);
}
onModuleInit() {
}
) {}
@bindThis
public async follow(
@ -56,7 +39,7 @@ export class ChannelFollowingService implements OnModuleInit {
followeeId: targetChannel.id,
});
this.globalEventService.publishInternalEvent('followChannel', {
await this.internalEventService.emit('followChannel', {
userId: requestUser.id,
channelId: targetChannel.id,
});
@ -72,37 +55,9 @@ export class ChannelFollowingService implements OnModuleInit {
followeeId: targetChannel.id,
});
this.globalEventService.publishInternalEvent('unfollowChannel', {
await this.internalEventService.emit('unfollowChannel', {
userId: requestUser.id,
channelId: targetChannel.id,
});
}
@bindThis
private async onMessage<E extends 'followChannel' | 'unfollowChannel'>(body: InternalEventTypes[E], type: E): Promise<void> {
{
switch (type) {
case 'followChannel': {
await this.userFollowingChannelsCache.delete(body.userId);
break;
}
case 'unfollowChannel': {
await this.userFollowingChannelsCache.delete(body.userId);
break;
}
}
}
}
@bindThis
public dispose(): void {
this.internalEventService.off('followChannel', this.onMessage);
this.internalEventService.off('unfollowChannel', this.onMessage);
this.userFollowingChannelsCache.dispose();
}
@bindThis
public onApplicationShutdown(signal?: string | undefined): void {
this.dispose();
}
}

8
packages/backend/src/core/ChatService.ts
View file

@ -28,6 +28,7 @@ import { CustomEmojiService } from '@/core/CustomEmojiService.js';
import { emojiRegex } from '@/misc/emoji-regex.js';
import { NotificationService } from '@/core/NotificationService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { TimeService } from '@/global/TimeService.js';
const MAX_ROOM_MEMBERS = 30;
const MAX_REACTIONS_PER_MESSAGE = 100;
@ -91,6 +92,7 @@ export class ChatService {
private userFollowingService: UserFollowingService,
private customEmojiService: CustomEmojiService,
private moderationLogService: ModerationLogService,
private readonly timeService: TimeService,
) {
}
@ -225,7 +227,7 @@ export class ChatService {
// 3秒経っても既読にならなかったらイベント発行
if (this.userEntityService.isLocalUser(toUser)) {
setTimeout(async () => {
this.timeService.startTimer(async () => {
const marker = await this.redisClient.get(`newUserChatMessageExists:${toUser.id}:${fromUser.id}`);
if (marker == null) return; // 既読
@ -285,7 +287,7 @@ export class ChatService {
redisPipeline.exec();
// 3秒経っても既読にならなかったらイベント発行
setTimeout(async () => {
this.timeService.startTimer(async () => {
const redisPipeline = this.redisClient.pipeline();
for (const membership of membershipsOtherThanMe) {
redisPipeline.get(`newRoomChatMessageExists:${membership.userId}:${toRoom.id}`);
@ -820,7 +822,7 @@ export class ChatService {
reaction = normalizeEmojiString(reaction_);
} else {
const name = custom[1];
const emoji = (await this.customEmojiService.localEmojisCache.fetch()).get(name);
const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(name);
if (emoji == null) {
throw new Error('no such emoji');

4
packages/backend/src/core/ClipService.ts
View file

@ -12,6 +12,7 @@ import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js
import { RoleService } from '@/core/RoleService.js';
import { IdService } from '@/core/IdService.js';
import type { MiLocalUser } from '@/models/User.js';
import { TimeService } from '@/global/TimeService.js';
@Injectable()
export class ClipService {
@ -33,6 +34,7 @@ export class ClipService {
private roleService: RoleService,
private idService: IdService,
private readonly timeService: TimeService,
) {
}
@ -125,7 +127,7 @@ export class ClipService {
}
this.clipsRepository.update(clip.id, {
lastClippedAt: new Date(),
lastClippedAt: this.timeService.date,
});
this.notesRepository.increment({ id: noteId }, 'clippedCount', 1);

32
packages/backend/src/core/CoreModule.ts
View file

@ -15,11 +15,10 @@ import { SystemWebhookService } from '@/core/SystemWebhookService.js';
import { UserSearchService } from '@/core/UserSearchService.js';
import { WebhookTestService } from '@/core/WebhookTestService.js';
import { FlashService } from '@/core/FlashService.js';
import { TimeService } from '@/core/TimeService.js';
import { EnvService } from '@/core/EnvService.js';
import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
import { ApLogService } from '@/core/ApLogService.js';
import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js';
import { InstanceStatsService } from '@/core/InstanceStatsService.js';
import { NoteVisibilityService } from '@/core/NoteVisibilityService.js';
import { AccountMoveService } from './AccountMoveService.js';
import { AccountUpdateService } from './AccountUpdateService.js';
@ -42,7 +41,6 @@ import { HttpRequestService } from './HttpRequestService.js';
import { IdService } from './IdService.js';
import { ImageProcessingService } from './ImageProcessingService.js';
import { SystemAccountService } from './SystemAccountService.js';
import { InternalEventService } from './InternalEventService.js';
import { InternalStorageService } from './InternalStorageService.js';
import { MetaService } from './MetaService.js';
import { MfmService } from './MfmService.js';
@ -159,12 +157,11 @@ import { ApPersonService } from './activitypub/models/ApPersonService.js';
import { ApQuestionService } from './activitypub/models/ApQuestionService.js';
import { QueueModule } from './QueueModule.js';
import { QueueService } from './QueueService.js';
import { LoggerService } from './LoggerService.js';
import { SponsorsService } from './SponsorsService.js';
import type { Provider } from '@nestjs/common';
import { GlobalModule } from '@/GlobalModule.js';
//#region 文字列ベースでのinjection用(循環参照対応のため)
const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService };
const $AbuseReportService: Provider = { provide: 'AbuseReportService', useExisting: AbuseReportService };
const $AbuseReportNotificationService: Provider = { provide: 'AbuseReportNotificationService', useExisting: AbuseReportNotificationService };
const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisting: AccountMoveService };
@ -188,7 +185,6 @@ const $HashtagService: Provider = { provide: 'HashtagService', useExisting: Hash
const $HttpRequestService: Provider = { provide: 'HttpRequestService', useExisting: HttpRequestService };
const $IdService: Provider = { provide: 'IdService', useExisting: IdService };
const $ImageProcessingService: Provider = { provide: 'ImageProcessingService', useExisting: ImageProcessingService };
const $InternalEventService: Provider = { provide: 'InternalEventService', useExisting: InternalEventService };
const $InternalStorageService: Provider = { provide: 'InternalStorageService', useExisting: InternalStorageService };
const $MetaService: Provider = { provide: 'MetaService', useExisting: MetaService };
const $MfmService: Provider = { provide: 'MfmService', useExisting: MfmService };
@ -239,8 +235,7 @@ const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService',
const $ChatService: Provider = { provide: 'ChatService', useExisting: ChatService };
const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService };
const $ReversiService: Provider = { provide: 'ReversiService', useExisting: ReversiService };
const $TimeService: Provider = { provide: 'TimeService', useExisting: TimeService };
const $EnvService: Provider = { provide: 'EnvService', useExisting: EnvService };
const $InstanceStatsService = { provide: 'InstanceStatsService', useExisting: InstanceStatsService };
const $NoteVisibilityService: Provider = { provide: 'NoteVisibilityService', useExisting: NoteVisibilityService };
const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService };
@ -322,10 +317,10 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
@Module({
imports: [
GlobalModule,
QueueModule,
],
providers: [
LoggerService,
AbuseReportService,
AbuseReportNotificationService,
AccountMoveService,
@ -349,7 +344,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
HttpRequestService,
IdService,
ImageProcessingService,
InternalEventService,
InternalStorageService,
MetaService,
MfmService,
@ -400,8 +394,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
ChatService,
RegistryApiService,
ReversiService,
TimeService,
EnvService,
InstanceStatsService,
NoteVisibilityService,
ChartLoggerService,
@ -482,7 +475,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
SponsorsService,
//#region 文字列ベースでのinjection用(循環参照対応のため)
$LoggerService,
$AbuseReportService,
$AbuseReportNotificationService,
$AccountMoveService,
@ -506,7 +498,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$HttpRequestService,
$IdService,
$ImageProcessingService,
$InternalEventService,
$InternalStorageService,
$MetaService,
$MfmService,
@ -557,8 +548,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$ChatService,
$RegistryApiService,
$ReversiService,
$TimeService,
$EnvService,
$InstanceStatsService,
$NoteVisibilityService,
$ChartLoggerService,
@ -640,7 +630,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
],
exports: [
QueueModule,
LoggerService,
AbuseReportService,
AbuseReportNotificationService,
AccountMoveService,
@ -664,7 +653,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
HttpRequestService,
IdService,
ImageProcessingService,
InternalEventService,
InternalStorageService,
MetaService,
MfmService,
@ -715,8 +703,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
ChatService,
RegistryApiService,
ReversiService,
TimeService,
EnvService,
InstanceStatsService,
NoteVisibilityService,
FederationChart,
@ -796,7 +783,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
SponsorsService,
//#region 文字列ベースでのinjection用(循環参照対応のため)
$LoggerService,
$AbuseReportService,
$AbuseReportNotificationService,
$AccountMoveService,
@ -820,7 +806,6 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$HttpRequestService,
$IdService,
$ImageProcessingService,
$InternalEventService,
$InternalStorageService,
$MetaService,
$MfmService,
@ -870,8 +855,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$ChatService,
$RegistryApiService,
$ReversiService,
$TimeService,
$EnvService,
$InstanceStatsService,
$NoteVisibilityService,
$FederationChart,

641
packages/backend/src/core/CustomEmojiService.ts
View file

@ -5,7 +5,7 @@
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import { In, IsNull } from 'typeorm';
import { In, IsNull, Not } from 'typeorm';
import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { IdService } from '@/core/IdService.js';
@ -14,12 +14,29 @@ import { bindThis } from '@/decorators.js';
import { DI } from '@/di-symbols.js';
import { MemoryKVCache, RedisSingleCache } from '@/misc/cache.js';
import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
import type { DriveFilesRepository, EmojisRepository, MiRole, MiUser } from '@/models/_.js';
import type { DriveFilesRepository, EmojisRepository, MiRole, MiUser, MiDriveFile, NotesRepository } from '@/models/_.js';
import type { MiEmoji } from '@/models/Emoji.js';
import type { Serialized } from '@/types.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import type { Config } from '@/config.js';
import { DriveService } from './DriveService.js';
import { DriveService } from '@/core/DriveService.js';
import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js';
import { TimeService } from '@/global/TimeService.js';
import { LoggerService } from '@/core/LoggerService.js';
import { isRetryableSymbol } from '@/misc/is-retryable-error.js';
import type Logger from '@/logger.js';
import { KeyNotFoundError } from '@/misc/errors/KeyNotFoundError.js';
// TODO move to sk-types.d.ts when merged
type MinEntity<T> = Omit<T, NullableProps<T>> & {
[K in NullableProps<T>]?: T[K] | undefined;
};
type SemiPartial<T, P extends keyof T> = Omit<T, P> & {
[Key in P]?: T[Key] | undefined;
};
type NullableProps<T> = {
[K in keyof T]: null extends T[K] ? K : never;
}[keyof T];
const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/;
@ -60,9 +77,15 @@ export const fetchEmojisSortKeys = [
export type FetchEmojisSortKeys = typeof fetchEmojisSortKeys[number];
@Injectable()
export class CustomEmojiService implements OnApplicationShutdown {
private emojisCache: MemoryKVCache<MiEmoji | null>;
public localEmojisCache: RedisSingleCache<Map<string, MiEmoji>>;
export class CustomEmojiService {
// id -> MiEmoji
public readonly emojisByIdCache: ManagedQuantumKVCache<MiEmoji>;
// key ("name host") -> MiEmoji (for remote emojis)
// key ("name") -> MiEmoji (for local emojis)
public readonly emojisByKeyCache: ManagedQuantumKVCache<MiEmoji>;
private readonly logger: Logger;
constructor(
@Inject(DI.redis)
@ -73,29 +96,55 @@ export class CustomEmojiService implements OnApplicationShutdown {
private emojisRepository: EmojisRepository,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private utilityService: UtilityService,
private idService: IdService,
private emojiEntityService: EmojiEntityService,
private moderationLogService: ModerationLogService,
private globalEventService: GlobalEventService,
private driveService: DriveService,
) {
this.emojisCache = new MemoryKVCache<MiEmoji | null>(1000 * 60 * 60 * 12); // 12h
private readonly timeService: TimeService,
this.localEmojisCache = new RedisSingleCache<Map<string, MiEmoji>>(this.redisClient, 'localEmojis', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60 * 3, // 3m
fetcher: () => this.emojisRepository.find({ where: { host: IsNull() } }).then(emojis => new Map(emojis.map(emoji => [emoji.name, emoji]))),
toRedisConverter: (value) => JSON.stringify(Array.from(value.values())),
fromRedisConverter: (value) => {
return new Map(JSON.parse(value).map((x: Serialized<MiEmoji>) => [x.name, {
...x,
updatedAt: x.updatedAt ? new Date(x.updatedAt) : null,
}]));
cacheManagementService: CacheManagementService,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('custom-emoji');
this.emojisByIdCache = cacheManagementService.createQuantumKVCache<MiEmoji>('emojisById', {
lifetime: 1000 * 60 * 60, // 1h
fetcher: async (id) => await this.emojisRepository.findOneByOrFail({ id }),
optionalFetcher: async (id) => await this.emojisRepository.findOneBy({ id }),
bulkFetcher: async (ids) => {
const emojis = await this.emojisRepository.findBy({ id: In(ids) });
return emojis.map(emoji => [emoji.id, emoji]);
},
});
this.emojisByKeyCache = cacheManagementService.createQuantumKVCache<MiEmoji>('emojisByKey', {
lifetime: 1000 * 60 * 60, // 1h
fetcher: async (key) => {
const { host, name } = decodeEmojiKey(key);
return await this.emojisRepository.findOneByOrFail({ host: host ?? IsNull(), name });
},
optionalFetcher: async (key) => {
const { host, name } = decodeEmojiKey(key);
return await this.emojisRepository.findOneBy({ host: host ?? IsNull(), name });
},
bulkFetcher: async (keys) => {
const queries = keys.map(key => {
const { host, name } = decodeEmojiKey(key);
return { host: host ?? IsNull(), name };
});
const emojis = await this.emojisRepository.findBy(queries);
return emojis.map(emoji => [encodeEmojiKey(emoji), emoji]);
},
});
}
/** @deprecated use createEmoji for new code */
@bindThis
public async add(data: {
originalUrl: string;
@ -110,31 +159,39 @@ export class CustomEmojiService implements OnApplicationShutdown {
localOnly: boolean;
roleIdsThatCanBeUsedThisEmojiAsReaction: MiRole['id'][];
}, moderator?: MiUser): Promise<MiEmoji> {
const emoji = await this.emojisRepository.insertOne({
id: this.idService.gen(),
updatedAt: new Date(),
name: data.name,
category: data.category,
host: data.host,
aliases: data.aliases,
originalUrl: data.originalUrl,
publicUrl: data.publicUrl,
type: data.fileType,
license: data.license,
isSensitive: data.isSensitive,
localOnly: data.localOnly,
roleIdsThatCanBeUsedThisEmojiAsReaction: data.roleIdsThatCanBeUsedThisEmojiAsReaction,
});
return await this.createEmoji(data, { moderator });
}
if (data.host == null) {
this.localEmojisCache.refresh();
public async createEmoji(
data: SemiPartial<MinEntity<MiEmoji>, 'id' | 'updatedAt' | 'aliases' | 'roleIdsThatCanBeUsedThisEmojiAsReaction'>,
opts?: { moderator?: { id: string } },
): Promise<MiEmoji> {
// Set defaults
data.id ??= this.idService.gen();
data.updatedAt ??= this.timeService.date;
data.aliases ??= [];
data.roleIdsThatCanBeUsedThisEmojiAsReaction ??= [];
this.globalEventService.publishBroadcastStream('emojiAdded', {
emoji: await this.emojiEntityService.packDetailed(emoji.id),
// Add to logs
this.logger.info(`Creating emoji name=${data.name} host=${data.host}...`);
// Add to database
await this.emojisRepository.insert(data);
// Add to cache
const emoji = await this.emojisByIdCache.fetch(data.id);
const emojiKey = encodeEmojiKey({ name: emoji.name, host: emoji.host });
this.emojisByIdCache.add(emojiKey, emoji); // This is a new entity, so we can use add() which does not emit sync events.
if (emoji.host == null) {
// Add to clients
await this.globalEventService.publishBroadcastStream('emojiAdded', {
emoji: await this.emojiEntityService.packDetailed(emoji),
});
if (moderator) {
this.moderationLogService.log(moderator, 'addCustomEmoji', {
// Add to mod logs
if (opts?.moderator) {
await this.moderationLogService.log(opts.moderator, 'addCustomEmoji', {
emojiId: emoji.id,
emoji: emoji,
});
@ -144,6 +201,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
return emoji;
}
/** @deprecated Use updateEmoji for new code */
@bindThis
public async update(data: (
{ id: MiEmoji['id'], name?: string; } | { name: string; id?: MiEmoji['id'], }
@ -161,91 +219,161 @@ export class CustomEmojiService implements OnApplicationShutdown {
null
| 'NO_SUCH_EMOJI'
| 'SAME_NAME_EMOJI_EXISTS'
> {
const emoji = data.id
? await this.getEmojiById(data.id)
: await this.getEmojiByName(data.name!);
if (emoji === null) return 'NO_SUCH_EMOJI';
const id = emoji.id;
> {
try {
const criteria = data.id
? { id: data.id as string }
: { name: data.name as string, host: null };
// IDと絵文字名が両方指定されている場合は絵文字名の変更を行うため重複チェックが必要
const doNameUpdate = data.id && data.name && (data.name !== emoji.name);
const updates = {
...data,
id: undefined,
host: undefined,
};
const opts = {
moderator,
};
await this.updateEmoji(criteria, updates, opts);
return null;
} catch (err) {
if (err instanceof KeyNotFoundError) return 'NO_SUCH_EMOJI';
if (err instanceof DuplicateEmojiError) return 'SAME_NAME_EMOJI_EXISTS';
throw err;
}
}
@bindThis
public async updateEmoji(
criteria: { id: string } | { name: string, host: string | null },
data: Omit<Partial<MiEmoji>, 'id' | 'host'>,
opts?: { moderator?: { id: string } },
): Promise<MiEmoji> {
const emoji = 'id' in criteria
? await this.emojisByIdCache.fetch(criteria.id)
: await this.emojisByKeyCache.fetch(encodeEmojiKey(criteria));
// Update the system logs
this.logger.info(`Updating emoji name=${emoji.name} host=${emoji.host}...`);
// If changing the name, then make sure we don't have a conflict.
const doNameUpdate = data.name !== undefined && data.name !== emoji.name;
if (doNameUpdate) {
const isDuplicate = await this.checkDuplicate(data.name!);
if (isDuplicate) return 'SAME_NAME_EMOJI_EXISTS';
const isDuplicate = await this.checkDuplicate(data.name as string, emoji.host);
if (isDuplicate) throw new DuplicateEmojiError(data.name as string, emoji.host);
}
await this.emojisRepository.update(emoji.id, {
updatedAt: new Date(),
name: data.name,
category: data.category,
aliases: data.aliases,
license: data.license,
isSensitive: data.isSensitive,
localOnly: data.localOnly,
originalUrl: data.originalUrl,
publicUrl: data.publicUrl,
type: data.fileType,
roleIdsThatCanBeUsedThisEmojiAsReaction: data.roleIdsThatCanBeUsedThisEmojiAsReaction ?? undefined,
});
// Make sure we always set the updated date!
data.updatedAt ??= this.timeService.date;
this.localEmojisCache.refresh();
// Update the database
await this.emojisRepository.update({ id: emoji.id }, data);
// If we're changing the file, then we need to delete the old one
if (data.originalUrl != null && data.originalUrl !== emoji.originalUrl) {
const oldFile = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() });
const newFile = await this.driveFilesRepository.findOneBy({ url: data.originalUrl, userHost: emoji.host ? emoji.host : IsNull() });
// Update the caches
const updated = await this.emojisByIdCache.refresh(emoji.id);
const updatedKey = encodeEmojiKey({ name: emoji.name, host: emoji.host });
await this.emojisByKeyCache.set(updatedKey, updated);
// But DON'T delete if this is the same file reference, otherwise we'll break the emoji!
if (oldFile && newFile && oldFile.id !== newFile.id) {
await this.driveService.deleteFile(oldFile, false, moderator ? moderator : undefined);
}
// Update the file
await this.updateEmojiFile(emoji, updated);
// If it's a remote emoji, then we're done.
// The remaining logic applies only to local emojis.
if (updated.host != null) {
return updated;
}
const packed = await this.emojiEntityService.packDetailed(emoji.id);
// Update the clients
if (!doNameUpdate) {
this.globalEventService.publishBroadcastStream('emojiUpdated', {
// If name is the same, then we can update in-place
const packed = await this.emojiEntityService.packDetailed(updated);
await this.globalEventService.publishBroadcastStream('emojiUpdated', {
emojis: [packed],
});
} else {
this.globalEventService.publishBroadcastStream('emojiDeleted', {
emojis: [await this.emojiEntityService.packDetailed(emoji)],
// If name has changed, we need to delete and recreate
const [oldPacked, newPacked] = await Promise.all([
this.emojiEntityService.packDetailed(emoji),
this.emojiEntityService.packDetailed(updated),
]);
await this.globalEventService.publishBroadcastStream('emojiDeleted', {
emojis: [oldPacked],
});
this.globalEventService.publishBroadcastStream('emojiAdded', {
emoji: packed,
await this.globalEventService.publishBroadcastStream('emojiAdded', {
emoji: newPacked,
});
}
if (moderator) {
const updated = await this.emojisRepository.findOneByOrFail({ id: id });
this.moderationLogService.log(moderator, 'updateCustomEmoji', {
// Update the mod logs
if (opts?.moderator) {
await this.moderationLogService.log(opts.moderator, 'updateCustomEmoji', {
emojiId: emoji.id,
before: emoji,
after: updated,
});
}
return null;
return updated;
}
@bindThis
private async updateEmojiFile(before: MiEmoji, after: MiEmoji, moderator?: { id: string }): Promise<void> {
// Nothing to do
if (after.originalUrl === before.originalUrl) {
return;
}
// If we're changing the file, then we need to delete the old one.
const [oldFile, newFile] = await Promise.all([
this.driveFilesRepository.findOneBy({ url: before.originalUrl, userHost: before.host ?? IsNull() }),
this.driveFilesRepository.findOneBy({ url: after.originalUrl, userHost: after.host ?? IsNull() }),
]);
// But DON'T delete if this is the same file reference, otherwise we'll break the emoji!
if (!oldFile || !newFile || oldFile.id === newFile.id) {
return;
}
await this.safeDeleteEmojiFile(before, oldFile, moderator);
}
@bindThis
private async safeDeleteEmojiFile(emoji: MiEmoji, file: MiDriveFile, moderator?: { id: string }): Promise<void> {
const [hasNoteReferences, hasEmojiReferences] = await Promise.all([
// Any note using this file ID is a reference.
this.notesRepository
.createQueryBuilder('note')
.where(':fileId <@ note.fileIds', { fileId: file.id })
.getExists(),
// Any *other* emoji using this file URL is a reference.
this.emojisRepository.existsBy({
originalUrl: file.url,
id: Not(emoji.id),
}),
]);
if (hasNoteReferences) {
this.logger.debug(`Not removing old file ${file.id} (${file.url}) - file is referenced by one or more notes.`);
} else if (hasEmojiReferences) {
this.logger.debug(`Not removing old file ${file.id} (${file.url}) - file is reference by another emoji.`);
} else {
this.logger.info(`Removing old file ${file.id} (${file.url}).`);
await this.driveService.deleteFile(file, false, moderator);
}
}
@bindThis
public async addAliasesBulk(ids: MiEmoji['id'][], aliases: string[]) {
const emojis = await this.emojisRepository.findBy({
id: In(ids),
});
for (const emoji of emojis) {
await this.emojisRepository.update(emoji.id, {
updatedAt: new Date(),
aliases: [...new Set(emoji.aliases.concat(aliases))],
});
}
this.localEmojisCache.refresh();
this.globalEventService.publishBroadcastStream('emojiUpdated', {
emojis: await this.emojiEntityService.packDetailedMany(ids),
await this.bulkUpdateEmojis(ids, async emojis => {
for (const emoji of emojis) {
await this.emojisRepository.update(emoji.id, {
updatedAt: this.timeService.date,
aliases: [...new Set(emoji.aliases.concat(aliases))],
});
}
});
}
@ -254,34 +382,22 @@ export class CustomEmojiService implements OnApplicationShutdown {
await this.emojisRepository.update({
id: In(ids),
}, {
updatedAt: new Date(),
updatedAt: this.timeService.date,
aliases: aliases,
});
this.localEmojisCache.refresh();
this.globalEventService.publishBroadcastStream('emojiUpdated', {
emojis: await this.emojiEntityService.packDetailedMany(ids),
});
await this.bulkUpdateEmojis(ids);
}
@bindThis
public async removeAliasesBulk(ids: MiEmoji['id'][], aliases: string[]) {
const emojis = await this.emojisRepository.findBy({
id: In(ids),
});
for (const emoji of emojis) {
await this.emojisRepository.update(emoji.id, {
updatedAt: new Date(),
aliases: emoji.aliases.filter(x => !aliases.includes(x)),
});
}
this.localEmojisCache.refresh();
this.globalEventService.publishBroadcastStream('emojiUpdated', {
emojis: await this.emojiEntityService.packDetailedMany(ids),
await this.bulkUpdateEmojis(ids, async emojis => {
for (const emoji of emojis) {
await this.emojisRepository.update(emoji.id, {
updatedAt: this.timeService.date,
aliases: emoji.aliases.filter(x => !aliases.includes(x)),
});
}
});
}
@ -290,15 +406,11 @@ export class CustomEmojiService implements OnApplicationShutdown {
await this.emojisRepository.update({
id: In(ids),
}, {
updatedAt: new Date(),
updatedAt: this.timeService.date,
category: category,
});
this.localEmojisCache.refresh();
this.globalEventService.publishBroadcastStream('emojiUpdated', {
emojis: await this.emojiEntityService.packDetailedMany(ids),
});
await this.bulkUpdateEmojis(ids);
}
@bindThis
@ -306,71 +418,115 @@ export class CustomEmojiService implements OnApplicationShutdown {
await this.emojisRepository.update({
id: In(ids),
}, {
updatedAt: new Date(),
updatedAt: this.timeService.date,
license: license,
});
this.localEmojisCache.refresh();
await this.bulkUpdateEmojis(ids);
}
this.globalEventService.publishBroadcastStream('emojiUpdated', {
emojis: await this.emojiEntityService.packDetailedMany(ids),
@bindThis
private async bulkUpdateEmojis(ids: MiEmoji['id'][], updater?: (emojis: readonly MiEmoji[]) => Promise<void>): Promise<void> {
// Update the database
if (updater) {
const emojis = await this.emojisByIdCache.fetchMany(ids);
await updater(emojis.values);
}
// Update the caches
const updated = await this.emojisByIdCache.refreshMany(ids);
const keyUpdates = updated.values.map(emoji => [encodeEmojiKey(emoji), emoji] as const);
await this.emojisByKeyCache.setMany(keyUpdates);
// Update the clients
await this.globalEventService.publishBroadcastStream('emojiUpdated', {
emojis: await this.emojiEntityService.packDetailedMany(updated.values),
});
}
@bindThis
public async delete(id: MiEmoji['id'], moderator?: MiUser) {
const emoji = await this.emojisRepository.findOneByOrFail({ id: id });
public async delete(id: MiEmoji['id'], moderator?: { id: string }) {
const emoji = await this.emojisByIdCache.fetch(id);
await this.emojisRepository.delete(emoji.id);
await Promise.all([
this.emojisRepository.delete(emoji.id),
this.emojisByIdCache.delete(emoji.id),
this.emojisByKeyCache.delete(encodeEmojiKey(emoji)),
]);
this.localEmojisCache.refresh();
const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() });
const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ?? IsNull() });
if (file) {
await this.driveService.deleteFile(file, false, moderator ? moderator : undefined);
await this.safeDeleteEmojiFile(emoji, file, moderator);
}
this.globalEventService.publishBroadcastStream('emojiDeleted', {
emojis: [await this.emojiEntityService.packDetailed(emoji)],
});
if (moderator) {
this.moderationLogService.log(moderator, 'deleteCustomEmoji', {
emojiId: emoji.id,
emoji: emoji,
if (emoji.host == null) {
await this.globalEventService.publishBroadcastStream('emojiDeleted', {
emojis: [await this.emojiEntityService.packDetailed(emoji)],
});
}
}
@bindThis
public async deleteBulk(ids: MiEmoji['id'][], moderator?: MiUser) {
const emojis = await this.emojisRepository.findBy({
id: In(ids),
});
for (const emoji of emojis) {
await this.emojisRepository.delete(emoji.id);
const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() });
if (file) {
await this.driveService.deleteFile(file, false, moderator ? moderator : undefined);
}
if (moderator) {
this.moderationLogService.log(moderator, 'deleteCustomEmoji', {
await this.moderationLogService.log(moderator, 'deleteCustomEmoji', {
emojiId: emoji.id,
emoji: emoji,
});
}
}
}
this.localEmojisCache.refresh();
@bindThis
public async deleteBulk(ids: MiEmoji['id'][], moderator?: MiUser) {
const emojis = await this.emojisByIdCache.fetchMany(ids);
this.globalEventService.publishBroadcastStream('emojiDeleted', {
emojis: await this.emojiEntityService.packDetailedMany(emojis),
});
const filesQueries = emojis.values.map(emoji => ({
url: emoji.originalUrl,
userHost: emoji.host ?? IsNull(),
}));
const files = await this.driveFilesRepository.findBy(filesQueries);
const emojiFiles = emojis.values
.map(emoji => {
const file = files.find(file => file.url === emoji.originalUrl && file.userHost === emoji.host);
return [emoji, file];
})
.filter(ef => ef[1] != null) as [MiEmoji, MiDriveFile][];
const localDeleted = emojis.values.filter(emoji => emoji.host == null);
const deletedKeys = emojis.values.map(emoji => encodeEmojiKey(emoji));
await Promise.all([
// Delete from database
this.emojisRepository.delete({ id: In(ids) }),
this.emojisByIdCache.deleteMany(ids),
// Delete from cache
this.emojisByKeyCache.deleteMany(deletedKeys),
// Delete from clients
localDeleted.length > 0
? this.emojiEntityService.packDetailedMany(localDeleted).then(async packed => {
await this.globalEventService.publishBroadcastStream('emojiDeleted', {
emojis: packed,
});
})
: null,
// Delete from mod logs
localDeleted.length > 0 && moderator != null
? Promise.all(localDeleted.map(async emoji => {
await this.moderationLogService.log(moderator, 'deleteCustomEmoji', {
emojiId: emoji.id,
emoji: emoji,
});
}))
: null,
// Delete from drive
emojiFiles.length > 0
? Promise.all(emojiFiles.map(async ([emoji, file]) => {
await this.safeDeleteEmojiFile(emoji, file, moderator);
}))
: null,
]);
}
@bindThis
@ -407,18 +563,10 @@ export class CustomEmojiService implements OnApplicationShutdown {
*/
@bindThis
public async populateEmoji(emojiName: string, noteUserHost: string | null): Promise<string | null> {
const { name, host } = this.parseEmojiStr(emojiName, noteUserHost);
if (name == null) return null;
if (host == null) return null;
const emojiKey = this.translateEmojiKey(emojiName, noteUserHost);
if (emojiKey == null) return null;
const newHost = host === this.config.host ? null : host;
const queryOrNull = async () => (await this.emojisRepository.findOneBy({
name,
host: newHost ?? IsNull(),
})) ?? null;
const emoji = await this.emojisCache.fetch(`${name} ${host}`, queryOrNull);
const emoji = await this.emojisByKeyCache.fetchMaybe(emojiKey);
if (emoji == null) return null;
return emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ）
@ -440,47 +588,45 @@ export class CustomEmojiService implements OnApplicationShutdown {
return res;
}
@bindThis
private translateEmojiKey(emojiName: string, noteUserHost: string | null): string | null {
const { name, host } = this.parseEmojiStr(emojiName, noteUserHost);
if (name == null) return null;
if (host == null) return null;
const newHost = host === this.config.host ? null : host;
return encodeEmojiKey({ name, host: newHost });
}
/**
* 与えられた絵文字のリストをデータベースから取得し、キャッシュに追加します
*/
@bindThis
public async prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise<void> {
const notCachedEmojis = emojis.filter(emoji => this.emojisCache.get(`${emoji.name} ${emoji.host}`) == null);
const emojisQuery: any[] = [];
const hosts = new Set(notCachedEmojis.map(e => e.host));
for (const host of hosts) {
if (host == null) continue;
emojisQuery.push({
name: In(notCachedEmojis.filter(e => e.host === host).map(e => e.name)),
host: host,
});
}
const _emojis = emojisQuery.length > 0 ? await this.emojisRepository.find({
where: emojisQuery,
select: ['name', 'host', 'originalUrl', 'publicUrl'],
}) : [];
for (const emoji of _emojis) {
this.emojisCache.set(`${emoji.name} ${emoji.host}`, emoji);
}
const emojiKeys = emojis.map(emoji => encodeEmojiKey(emoji));
await this.emojisByKeyCache.fetchMany(emojiKeys);
}
/**
* ローカル内の絵文字に重複がないかチェックします
* @param name 絵文字名
* @param host Emoji hostname
*/
@bindThis
public checkDuplicate(name: string): Promise<boolean> {
return this.emojisRepository.exists({ where: { name, host: IsNull() } });
public async checkDuplicate(name: string, host: string | null = null): Promise<boolean> {
const emoji = await this.getEmojiByName(name, host);
return emoji != null;
}
@bindThis
public getEmojiById(id: string): Promise<MiEmoji | null> {
return this.emojisRepository.findOneBy({ id });
public async getEmojiById(id: string): Promise<MiEmoji | null> {
return await this.emojisByIdCache.fetchMaybe(id) ?? null;
}
@bindThis
public getEmojiByName(name: string): Promise<MiEmoji | null> {
return this.emojisRepository.findOneBy({ name, host: IsNull() });
public async getEmojiByName(name: string, host: string | null = null): Promise<MiEmoji | null> {
const emojiKey = encodeEmojiKey({ name, host });
return await this.emojisByKeyCache.fetchMaybe(emojiKey) ?? null;
}
@bindThis
@ -627,14 +773,97 @@ export class CustomEmojiService implements OnApplicationShutdown {
allPages: Math.ceil(count / limit),
};
}
}
@bindThis
public dispose(): void {
this.emojisCache.dispose();
}
export class InvalidEmojiError extends Error {
// Fix the error name in stack traces - https://stackoverflow.com/a/71573071
override name = this.constructor.name;
@bindThis
public onApplicationShutdown(signal?: string | undefined): void {
this.dispose();
public readonly [isRetryableSymbol] = false;
}
export class InvalidEmojiKeyError extends InvalidEmojiError {
constructor(
public readonly key: string,
message?: string,
) {
const actualMessage = message
? `Invalid emoji key "${key}": ${message}`
: `Invalid emoji key "${key}".`;
super(actualMessage);
}
}
export class InvalidEmojiNameError extends InvalidEmojiError {
constructor(
public readonly name: string,
message?: string,
) {
const actualMessage = message
? `Invalid emoji name "${name}": ${message}`
: `Invalid emoji name "${name}".`;
super(actualMessage);
}
}
export class InvalidEmojiHostError extends InvalidEmojiError {
constructor(
public readonly host: string | null,
message?: string,
) {
const hostString = host == null ? 'null' : `"${host}"`;
const actualMessage = message
? `Invalid emoji name ${hostString}: ${message}`
: `Invalid emoji name ${hostString}.`;
super(actualMessage);
}
}
export class DuplicateEmojiError extends InvalidEmojiError {
constructor(
public readonly name: string,
public readonly host: string | null,
message?: string,
) {
const hostString = host == null ? 'null' : `"${host}"`;
const actualMessage = message
? `Duplicate emoji name "${name}" for host ${hostString}: ${message}`
: `Duplicate emoji name "${name}" for host ${hostString}.`;
super(actualMessage);
}
}
export function isValidEmojiName(name: string): boolean {
return name !== '' && !name.includes(' ');
}
export function isValidEmojiHost(host: string): boolean {
return host !== '' && !host.includes(' ');
}
// TODO unit tests
export function encodeEmojiKey(emoji: { name: string, host: string | null }): string {
if (emoji.name === '') throw new InvalidEmojiNameError(emoji.name, 'Name cannot be empty.');
if (emoji.name.includes(' ')) throw new InvalidEmojiNameError(emoji.name, 'Name cannot contain a space.');
// Local emojis are just the name.
if (emoji.host == null) {
return emoji.name;
}
if (emoji.host === '') throw new InvalidEmojiHostError(emoji.host, 'Host cannot be empty.');
if (emoji.host.includes(' ')) throw new InvalidEmojiHostError(emoji.host, 'Host cannot contain a space.');
return `${emoji.name} ${emoji.host}`;
}
// TODO unit tests
export function decodeEmojiKey(key: string): { name: string, host: string | null } {
const match = key.match(/^([^ ]+)(?: ([^ ]+))?$/);
if (!match) {
throw new InvalidEmojiKeyError(key);
}
const name = match[1];
const host = match[2] || null;
return { name, host };
}

26
packages/backend/src/core/DriveService.ts
View file

@ -424,10 +424,10 @@ export class DriveService {
if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read';
try {
if (this.bunnyService.usingBunnyCDN(this.meta)) {
await this.bunnyService.upload(this.meta, key, stream);
if (this.bunnyService.usingBunnyCDN()) {
await this.bunnyService.upload(key, stream);
} else {
const result = await this.s3Service.upload(this.meta, params);
const result = await this.s3Service.upload(params);
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
} else { // AbortMultipartUploadCommandOutput
@ -739,7 +739,7 @@ export class DriveService {
}
@bindThis
public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: MiUser) {
public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: { id: string }) {
if (file.storedInternal) {
this.deleteLocalFile(file.accessKey!);
@ -766,8 +766,8 @@ export class DriveService {
}
@bindThis
public async deleteFileSync(file: MiDriveFile, isExpired = false, deleter?: MiUser) {
const promises = [];
public async deleteFileSync(file: MiDriveFile, isExpired = false, deleter?: { id: string }) {
const promises: Promise<void>[] = [];
if (file.storedInternal) {
promises.push(this.deleteLocalFile(file.accessKey!));
@ -797,7 +797,7 @@ export class DriveService {
}
@bindThis
private async deletePostProcess(file: MiDriveFile, isExpired = false, deleter?: MiUser) {
private async deletePostProcess(file: MiDriveFile, isExpired = false, deleter?: { id: string }) {
// リモートファイル期限切れ削除後は直リンクにする
if (isExpired && file.userHost !== null && file.uri != null) {
this.driveFilesRepository.update(file.id, {
@ -843,15 +843,17 @@ export class DriveService {
@bindThis
public async deleteObjectStorageFile(key: string) {
try {
if (this.bunnyService.usingBunnyCDN()) {
await this.bunnyService.delete(key);
return;
}
const param = {
Bucket: this.meta.objectStorageBucket,
Key: key,
} as DeleteObjectCommandInput;
if (this.bunnyService.usingBunnyCDN(this.meta)) {
await this.bunnyService.delete(this.meta, key);
} else {
await this.s3Service.delete(this.meta, param);
}
await this.s3Service.delete(param);
} catch (err: any) {
if (err.name === 'NoSuchKey') {
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`);

20
packages/backend/src/core/EnvService.ts
View file

@ -1,20 +0,0 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Injectable } from '@nestjs/common';
/**
* Provides access to the process environment variables.
* This exists for testing purposes, so that a test can mock the environment without corrupting state for other tests.
*/
@Injectable()
export class EnvService {
/**
* Passthrough to process.env
*/
public get env() {
return process.env;
}
}

2
packages/backend/src/core/FanoutTimelineEndpointService.ts
View file

@ -276,7 +276,7 @@ export class FanoutTimelineEndpointService {
// Fetch everything and populate users
const [users, instances] = await Promise.all([
this.cacheService.getUsers(usersToFetch),
this.cacheService.findUsersById(usersToFetch),
this.federatedInstanceService.federatedInstanceCache.fetchMany(instancesToFetch).then(i => new Map(i)),
]);
for (const [id, user] of Array.from(users)) {

4
packages/backend/src/core/FanoutTimelineService.ts
View file

@ -8,6 +8,7 @@ import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
import { TimeService } from '@/global/TimeService.js';
export type FanoutTimelineName = (
// home timeline
@ -46,6 +47,7 @@ export class FanoutTimelineService {
private redisForTimelines: Redis.Redis,
private idService: IdService,
private readonly timeService: TimeService,
) {
}
@ -53,7 +55,7 @@ export class FanoutTimelineService {
public push(tl: FanoutTimelineName, id: string, maxlen: number, pipeline: Redis.ChainableCommander) {
// リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、
// 3分以内に投稿されたものでない場合、Redisにある最古のIDより新しい場合のみ追加する
if (this.idService.parse(id).date.getTime() > Date.now() - 1000 * 60 * 3) {
if (this.idService.parse(id).date.getTime() > this.timeService.now - 1000 * 60 * 3) {
pipeline.lpush('list:' + tl, id);
if (Math.random() < 0.1) { // 10%の確率でトリム
pipeline.ltrim('list:' + tl, 0, maxlen - 1);

4
packages/backend/src/core/FeaturedService.ts
View file

@ -9,6 +9,7 @@ import type { MiGalleryPost, MiNote, MiUser } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import { TimeService } from '@/global/TimeService.js';
const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
export const GALLERY_POSTS_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
@ -24,12 +25,13 @@ export class FeaturedService {
private redisClient: Redis.Redis, // TODO: 専用のRedisサーバーを設定できるようにする
private readonly roleService: RoleService,
private readonly timeService: TimeService,
) {
}
@bindThis
private getCurrentWindow(windowRange: number): number {
const passed = new Date().getTime() - featuredEpoc;
const passed = this.timeService.now - featuredEpoc;
return Math.floor(passed / windowRange);
}

50
packages/backend/src/core/FederatedInstanceService.ts
View file

@ -4,22 +4,24 @@
*/
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import { In } from 'typeorm';
import type { InstancesRepository, MiMeta } from '@/models/_.js';
import type { InstancesRepository } from '@/models/_.js';
import type { MiMeta } from '@/models/Meta.js';
import type { MiInstance } from '@/models/Instance.js';
import type { InternalEventTypes } from '@/core/GlobalEventService.js';
import { IdService } from '@/core/IdService.js';
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import { diffArraysSimple } from '@/misc/diff-arrays.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import { bindThis } from '@/decorators.js';
import { TimeService } from '@/global/TimeService.js';
import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
@Injectable()
export class FederatedInstanceService implements OnApplicationShutdown {
public readonly federatedInstanceCache: QuantumKVCache<MiInstance>;
public readonly federatedInstanceCache: ManagedQuantumKVCache<MiInstance>;
constructor(
@Inject(DI.instancesRepository)
@ -31,8 +33,12 @@ export class FederatedInstanceService implements OnApplicationShutdown {
private utilityService: UtilityService,
private idService: IdService,
private readonly internalEventService: InternalEventService,
private readonly timeService: TimeService,
cacheManagementService: CacheManagementService,
) {
this.federatedInstanceCache = new QuantumKVCache(this.internalEventService, 'federatedInstance', {
this.federatedInstanceCache = cacheManagementService.createQuantumKVCache('federatedInstance', {
// TODO can we increase this?
lifetime: 1000 * 60 * 3, // 3 minutes
fetcher: async key => {
const host = this.utilityService.toPuny(key);
@ -43,7 +49,7 @@ export class FederatedInstanceService implements OnApplicationShutdown {
.values({
id: this.idService.gen(),
host,
firstRetrievedAt: new Date(),
firstRetrievedAt: this.timeService.date,
isBlocked: this.utilityService.isBlockedHost(host),
isSilenced: this.utilityService.isSilencedHost(host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(host),
@ -57,14 +63,15 @@ export class FederatedInstanceService implements OnApplicationShutdown {
}
return instance;
},
// optionalFetcher not needed
bulkFetcher: async keys => {
const hosts = keys.map(key => this.utilityService.toPuny(key));
const instances = await this.instancesRepository.findBy({ host: In(hosts) });
return instances.map(i => [i.host, i]);
return instances.map(instance => [instance.host, instance]);
},
});
this.internalEventService.on('metaUpdated', this.onMetaUpdated);
this.internalEventService.on('metaUpdated', this.onMetaUpdated, { ignoreRemote: true });
}
@bindThis
@ -83,7 +90,7 @@ export class FederatedInstanceService implements OnApplicationShutdown {
.values({
id: this.idService.gen(),
host,
firstRetrievedAt: new Date(),
firstRetrievedAt: this.timeService.date,
isBlocked: this.utilityService.isBlockedHost(host),
isSilenced: this.utilityService.isSilencedHost(host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(host),
@ -159,34 +166,31 @@ export class FederatedInstanceService implements OnApplicationShutdown {
return instances.map(i => i[1]);
}
// This gets fired *in each process* so don't do anything to trigger cache notifications!
private syncCache(before: MiMeta | undefined, after: MiMeta): void {
const changed =
@bindThis
private async onMetaUpdated(body: InternalEventTypes['metaUpdated']): Promise<void> {
const { before, after } = body;
const changed = (
diffArraysSimple(before?.blockedHosts, after.blockedHosts) ||
diffArraysSimple(before?.silencedHosts, after.silencedHosts) ||
diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) ||
diffArraysSimple(before?.federationHosts, after.federationHosts) ||
diffArraysSimple(before?.bubbleInstances, after.bubbleInstances);
diffArraysSimple(before?.bubbleInstances, after.bubbleInstances)
);
if (changed) {
// We have to clear the whole thing, otherwise subdomains won't be synced.
// This gets fired in *each* process so don't do anything to trigger cache notifications!
this.federatedInstanceCache.clear();
}
}
@bindThis
private async onMetaUpdated(body: { before?: MiMeta; after: MiMeta; }) {
this.syncCache(body.before, body.after);
}
@bindThis
public dispose(): void {
public dispose() {
this.internalEventService.off('metaUpdated', this.onMetaUpdated);
this.federatedInstanceCache.dispose();
}
@bindThis
public onApplicationShutdown(signal?: string | undefined): void {
public onApplicationShutdown() {
this.dispose();
}
}

7
packages/backend/src/core/FetchInstanceMetadataService.ts
View file

@ -15,6 +15,7 @@ import { LoggerService } from '@/core/LoggerService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { TimeService } from '@/global/TimeService.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import type { CheerioAPI } from 'cheerio/slim';
@ -47,6 +48,8 @@ export class FetchInstanceMetadataService {
private federatedInstanceService: FederatedInstanceService,
@Inject(DI.redis)
private redisClient: Redis.Redis,
private readonly timeService: TimeService,
) {
this.logger = this.loggerService.getLogger('metadata', 'cyan');
}
@ -84,7 +87,7 @@ export class FetchInstanceMetadataService {
try {
if (!force) {
const _instance = await this.federatedInstanceService.fetchOrRegister(host);
const now = Date.now();
const now = this.timeService.now;
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
// unlock at the finally caluse
return;
@ -110,7 +113,7 @@ export class FetchInstanceMetadataService {
this.logger.debug(`Successfuly fetched metadata of ${instance.host}`);
const updates = {
infoUpdatedAt: new Date(),
infoUpdatedAt: this.timeService.date,
} as Record<string, any>;
if (info) {

2
packages/backend/src/core/FileInfoService.ts
View file

@ -143,7 +143,7 @@ export class FileInfoService {
}
@bindThis
public fixMime(mime: string | fileType.MimeType): string {
public fixMime(mime: string): string {
// see https://github.com/misskey-dev/misskey/pull/10686
if (mime === 'audio/x-flac') {
return 'audio/flac';

99
packages/backend/src/core/GlobalEventService.ts
View file

@ -20,12 +20,14 @@ import type { MiPage } from '@/models/Page.js';
import type { MiWebhook } from '@/models/Webhook.js';
import type { MiSystemWebhook } from '@/models/SystemWebhook.js';
import type { MiMeta } from '@/models/Meta.js';
import { MiAvatarDecoration, MiChatMessage, MiChatRoom, MiReversiGame, MiRole, MiRoleAssignment } from '@/models/_.js';
import type { MiAvatarDecoration, MiChatMessage, MiChatRoom, MiReversiGame, MiRole, MiRoleAssignment } from '@/models/_.js';
import type { Packed } from '@/misc/json-schema.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
import { Serialized } from '@/types.js';
import type { Serialized } from '@/types.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import { trackPromise } from '@/misc/promise-tracker.js';
import type Emitter from 'strict-event-emitter-types';
import type { EventEmitter } from 'events';
@ -233,8 +235,12 @@ export interface InternalEventTypes {
userChangeSuspendedState: { id: MiUser['id']; isSuspended: MiUser['isSuspended']; };
userChangeDeletedState: { id: MiUser['id']; isDeleted: MiUser['isDeleted']; };
userTokenRegenerated: { id: MiUser['id']; oldToken: string; newToken: string; };
/** @deprecated Use userUpdated or usersUpdated instead */
remoteUserUpdated: { id: MiUser['id']; };
/** @deprecated Use userUpdated or usersUpdated instead */
localUserUpdated: { id: MiUser['id']; };
usersUpdated: { ids: MiUser['id'][]; };
userUpdated: { id: MiUser['id']; };
follow: { followerId: MiUser['id']; followeeId: MiUser['id']; };
unfollow: { followerId: MiUser['id']; followeeId: MiUser['id']; };
blockingCreated: { blockerId: MiUser['id']; blockeeId: MiUser['id']; };
@ -245,12 +251,12 @@ export interface InternalEventTypes {
roleUpdated: MiRole;
userRoleAssigned: MiRoleAssignment;
userRoleUnassigned: MiRoleAssignment;
webhookCreated: MiWebhook;
webhookDeleted: MiWebhook;
webhookUpdated: MiWebhook;
systemWebhookCreated: MiSystemWebhook;
systemWebhookDeleted: MiSystemWebhook;
systemWebhookUpdated: MiSystemWebhook;
webhookCreated: { id: MiWebhook['id'] };
webhookDeleted: { id: MiWebhook['id'] };
webhookUpdated: { id: MiWebhook['id'] };
systemWebhookCreated: { id: MiSystemWebhook['id'] };
systemWebhookDeleted: { id: MiSystemWebhook['id'] };
systemWebhookUpdated: { id: MiSystemWebhook['id'] };
antennaCreated: MiAntenna;
antennaDeleted: MiAntenna;
antennaUpdated: MiAntenna;
@ -260,12 +266,17 @@ export interface InternalEventTypes {
metaUpdated: { before?: MiMeta; after: MiMeta; };
followChannel: { userId: MiUser['id']; channelId: MiChannel['id']; };
unfollowChannel: { userId: MiUser['id']; channelId: MiChannel['id']; };
updateUserProfile: MiUserProfile;
updateUserProfile: { userId: MiUserProfile['userId'] };
mute: { muterId: MiUser['id']; muteeId: MiUser['id']; };
unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; };
userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; };
userListMemberUpdated: { userListId: MiUserList['id']; memberId: MiUser['id']; };
userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; };
userListMemberBulkAdded: { userListIds: MiUserList['id'][]; memberId: MiUser['id']; };
userListMemberBulkUpdated: { userListIds: MiUserList['id'][]; memberId: MiUser['id']; };
userListMemberBulkRemoved: { userListIds: MiUserList['id'][]; memberId: MiUser['id']; };
quantumCacheUpdated: { name: string, keys: string[] };
quantumCacheReset: { name: string };
}
type EventTypesToEventPayload<T> = EventUnionFromDictionary<UndefinedAsNullAll<SerializedAll<T>>>;
@ -350,6 +361,8 @@ export class GlobalEventService {
@Inject(DI.redisForPub)
private redisForPub: Redis.Redis,
private readonly internalEventService: InternalEventService,
) {
}
@ -365,81 +378,83 @@ export class GlobalEventService {
}));
}
/** @deprecated use InternalEventService instead */
@bindThis
public publishInternalEvent<K extends keyof InternalEventTypes>(type: K, value?: InternalEventTypes[K]): void {
this.publish('internal', type, typeof value === 'undefined' ? null : value);
public publishInternalEvent<K extends keyof InternalEventTypes>(type: K, value: InternalEventTypes[K]): void {
trackPromise(this.internalEventService.emit(type, value));
}
/** @deprecated use InternalEventService instead */
@bindThis
public async publishInternalEventAsync<K extends keyof InternalEventTypes>(type: K, value: InternalEventTypes[K]): Promise<void> {
await this.internalEventService.emit(type, value);
}
@bindThis
public async publishInternalEventAsync<K extends keyof InternalEventTypes>(type: K, value?: InternalEventTypes[K]): Promise<void> {
await this.publish('internal', type, typeof value === 'undefined' ? null : value);
public async publishBroadcastStream<K extends keyof BroadcastTypes>(type: K, value?: BroadcastTypes[K]): Promise<void> {
await this.publish('broadcast', type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishBroadcastStream<K extends keyof BroadcastTypes>(type: K, value?: BroadcastTypes[K]): void {
this.publish('broadcast', type, typeof value === 'undefined' ? null : value);
public async publishMainStream<K extends keyof MainEventTypes>(userId: MiUser['id'], type: K, value?: MainEventTypes[K]): Promise<void> {
await this.publish(`mainStream:${userId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishMainStream<K extends keyof MainEventTypes>(userId: MiUser['id'], type: K, value?: MainEventTypes[K]): void {
this.publish(`mainStream:${userId}`, type, typeof value === 'undefined' ? null : value);
public async publishDriveStream<K extends keyof DriveEventTypes>(userId: MiUser['id'], type: K, value?: DriveEventTypes[K]): Promise<void> {
await this.publish(`driveStream:${userId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishDriveStream<K extends keyof DriveEventTypes>(userId: MiUser['id'], type: K, value?: DriveEventTypes[K]): void {
this.publish(`driveStream:${userId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishNoteStream<K extends keyof NoteEventTypes>(noteId: MiNote['id'], type: K, value?: NoteEventTypes[K]): void {
this.publish(`noteStream:${noteId}`, type, {
public async publishNoteStream<K extends keyof NoteEventTypes>(noteId: MiNote['id'], type: K, value?: NoteEventTypes[K]): Promise<void> {
await this.publish(`noteStream:${noteId}`, type, {
id: noteId,
body: value,
});
}
@bindThis
public publishUserListStream<K extends keyof UserListEventTypes>(listId: MiUserList['id'], type: K, value?: UserListEventTypes[K]): void {
this.publish(`userListStream:${listId}`, type, typeof value === 'undefined' ? null : value);
public async publishUserListStream<K extends keyof UserListEventTypes>(listId: MiUserList['id'], type: K, value?: UserListEventTypes[K]): Promise<void> {
await this.publish(`userListStream:${listId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishAntennaStream<K extends keyof AntennaEventTypes>(antennaId: MiAntenna['id'], type: K, value?: AntennaEventTypes[K]): void {
this.publish(`antennaStream:${antennaId}`, type, typeof value === 'undefined' ? null : value);
public async publishAntennaStream<K extends keyof AntennaEventTypes>(antennaId: MiAntenna['id'], type: K, value?: AntennaEventTypes[K]): Promise<void> {
await this.publish(`antennaStream:${antennaId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishRoleTimelineStream<K extends keyof RoleTimelineEventTypes>(roleId: MiRole['id'], type: K, value?: RoleTimelineEventTypes[K]): void {
this.publish(`roleTimelineStream:${roleId}`, type, typeof value === 'undefined' ? null : value);
public async publishRoleTimelineStream<K extends keyof RoleTimelineEventTypes>(roleId: MiRole['id'], type: K, value?: RoleTimelineEventTypes[K]): Promise<void> {
await this.publish(`roleTimelineStream:${roleId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishNotesStream(note: Packed<'Note'>): void {
this.publish('notesStream', null, note);
public async publishNotesStream(note: Packed<'Note'>): Promise<void> {
await this.publish('notesStream', null, note);
}
@bindThis
public publishAdminStream<K extends keyof AdminEventTypes>(userId: MiUser['id'], type: K, value?: AdminEventTypes[K]): void {
this.publish(`adminStream:${userId}`, type, typeof value === 'undefined' ? null : value);
public async publishAdminStream<K extends keyof AdminEventTypes>(userId: MiUser['id'], type: K, value?: AdminEventTypes[K]): Promise<void> {
await this.publish(`adminStream:${userId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishChatUserStream<K extends keyof ChatEventTypes>(fromUserId: MiUser['id'], toUserId: MiUser['id'], type: K, value?: ChatEventTypes[K]): void {
this.publish(`chatUserStream:${fromUserId}-${toUserId}`, type, typeof value === 'undefined' ? null : value);
public async publishChatUserStream<K extends keyof ChatEventTypes>(fromUserId: MiUser['id'], toUserId: MiUser['id'], type: K, value?: ChatEventTypes[K]): Promise<void> {
await this.publish(`chatUserStream:${fromUserId}-${toUserId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishChatRoomStream<K extends keyof ChatEventTypes>(toRoomId: MiChatRoom['id'], type: K, value?: ChatEventTypes[K]): void {
this.publish(`chatRoomStream:${toRoomId}`, type, typeof value === 'undefined' ? null : value);
public async publishChatRoomStream<K extends keyof ChatEventTypes>(toRoomId: MiChatRoom['id'], type: K, value?: ChatEventTypes[K]): Promise<void> {
await this.publish(`chatRoomStream:${toRoomId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishReversiStream<K extends keyof ReversiEventTypes>(userId: MiUser['id'], type: K, value?: ReversiEventTypes[K]): void {
this.publish(`reversiStream:${userId}`, type, typeof value === 'undefined' ? null : value);
public async publishReversiStream<K extends keyof ReversiEventTypes>(userId: MiUser['id'], type: K, value?: ReversiEventTypes[K]): Promise<void> {
await this.publish(`reversiStream:${userId}`, type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishReversiGameStream<K extends keyof ReversiGameEventTypes>(gameId: MiReversiGame['id'], type: K, value?: ReversiGameEventTypes[K]): void {
this.publish(`reversiGameStream:${gameId}`, type, typeof value === 'undefined' ? null : value);
public async publishReversiGameStream<K extends keyof ReversiGameEventTypes>(gameId: MiReversiGame['id'], type: K, value?: ReversiGameEventTypes[K]): Promise<void> {
await this.publish(`reversiGameStream:${gameId}`, type, typeof value === 'undefined' ? null : value);
}
}

37
packages/backend/src/core/HashtagService.ts
View file

@ -7,14 +7,15 @@ import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import type { MiUser } from '@/models/User.js';
import { isLocalUser, isRemoteUser } from '@/models/User.js';
import { normalizeForSearch } from '@/misc/normalize-for-search.js';
import { IdService } from '@/core/IdService.js';
import type { MiHashtag } from '@/models/Hashtag.js';
import type { HashtagsRepository, MiMeta } from '@/models/_.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { TimeService } from '@/global/TimeService.js';
@Injectable()
export class HashtagService {
@ -28,10 +29,10 @@ export class HashtagService {
@Inject(DI.hashtagsRepository)
private hashtagsRepository: HashtagsRepository,
private userEntityService: UserEntityService,
private featuredService: FeaturedService,
private idService: IdService,
private utilityService: UtilityService,
private readonly timeService: TimeService,
) {
}
@ -78,19 +79,19 @@ export class HashtagService {
set.attachedUsersCount = () => '"attachedUsersCount" + 1';
}
// 自分が(ローカル内で)初めてこのタグを使ったなら
if (this.userEntityService.isLocalUser(user) && !index.attachedLocalUserIds.some(id => id === user.id)) {
if (isLocalUser(user) && !index.attachedLocalUserIds.some(id => id === user.id)) {
set.attachedLocalUserIds = () => `array_append("attachedLocalUserIds", '${user.id}')`;
set.attachedLocalUsersCount = () => '"attachedLocalUsersCount" + 1';
}
// 自分が(リモートで)初めてこのタグを使ったなら
if (this.userEntityService.isRemoteUser(user) && !index.attachedRemoteUserIds.some(id => id === user.id)) {
if (isRemoteUser(user) && !index.attachedRemoteUserIds.some(id => id === user.id)) {
set.attachedRemoteUserIds = () => `array_append("attachedRemoteUserIds", '${user.id}')`;
set.attachedRemoteUsersCount = () => '"attachedRemoteUsersCount" + 1';
}
} else {
set.attachedUserIds = () => `array_remove("attachedUserIds", '${user.id}')`;
set.attachedUsersCount = () => '"attachedUsersCount" - 1';
if (this.userEntityService.isLocalUser(user)) {
if (isLocalUser(user)) {
set.attachedLocalUserIds = () => `array_remove("attachedLocalUserIds", '${user.id}')`;
set.attachedLocalUsersCount = () => '"attachedLocalUsersCount" - 1';
} else {
@ -105,12 +106,12 @@ export class HashtagService {
set.mentionedUsersCount = () => '"mentionedUsersCount" + 1';
}
// 自分が(ローカル内で)初めてこのタグを使ったなら
if (this.userEntityService.isLocalUser(user) && !index.mentionedLocalUserIds.some(id => id === user.id)) {
if (isLocalUser(user) && !index.mentionedLocalUserIds.some(id => id === user.id)) {
set.mentionedLocalUserIds = () => `array_append("mentionedLocalUserIds", '${user.id}')`;
set.mentionedLocalUsersCount = () => '"mentionedLocalUsersCount" + 1';
}
// 自分が(リモートで)初めてこのタグを使ったなら
if (this.userEntityService.isRemoteUser(user) && !index.mentionedRemoteUserIds.some(id => id === user.id)) {
if (isRemoteUser(user) && !index.mentionedRemoteUserIds.some(id => id === user.id)) {
set.mentionedRemoteUserIds = () => `array_append("mentionedRemoteUserIds", '${user.id}')`;
set.mentionedRemoteUsersCount = () => '"mentionedRemoteUsersCount" + 1';
}
@ -133,10 +134,10 @@ export class HashtagService {
mentionedRemoteUsersCount: 0,
attachedUserIds: [user.id],
attachedUsersCount: 1,
attachedLocalUserIds: this.userEntityService.isLocalUser(user) ? [user.id] : [],
attachedLocalUsersCount: this.userEntityService.isLocalUser(user) ? 1 : 0,
attachedRemoteUserIds: this.userEntityService.isRemoteUser(user) ? [user.id] : [],
attachedRemoteUsersCount: this.userEntityService.isRemoteUser(user) ? 1 : 0,
attachedLocalUserIds: isLocalUser(user) ? [user.id] : [],
attachedLocalUsersCount: isLocalUser(user) ? 1 : 0,
attachedRemoteUserIds: isRemoteUser(user) ? [user.id] : [],
attachedRemoteUsersCount: isRemoteUser(user) ? 1 : 0,
} as MiHashtag);
} else {
this.hashtagsRepository.insert({
@ -144,10 +145,10 @@ export class HashtagService {
name: tag,
mentionedUserIds: [user.id],
mentionedUsersCount: 1,
mentionedLocalUserIds: this.userEntityService.isLocalUser(user) ? [user.id] : [],
mentionedLocalUsersCount: this.userEntityService.isLocalUser(user) ? 1 : 0,
mentionedRemoteUserIds: this.userEntityService.isRemoteUser(user) ? [user.id] : [],
mentionedRemoteUsersCount: this.userEntityService.isRemoteUser(user) ? 1 : 0,
mentionedLocalUserIds: isLocalUser(user) ? [user.id] : [],
mentionedLocalUsersCount: isLocalUser(user) ? 1 : 0,
mentionedRemoteUserIds: isRemoteUser(user) ? [user.id] : [],
mentionedRemoteUsersCount: isRemoteUser(user) ? 1 : 0,
attachedUserIds: [],
attachedUsersCount: 0,
attachedLocalUserIds: [],
@ -166,7 +167,7 @@ export class HashtagService {
if (this.utilityService.isKeyWordIncluded(hashtag, this.meta.sensitiveWords)) return;
// YYYYMMDDHHmm (10分間隔)
const now = new Date();
const now = this.timeService.date;
now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0);
const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`;
@ -197,7 +198,7 @@ export class HashtagService {
@bindThis
public async getChart(hashtag: string, range: number): Promise<number[]> {
const now = new Date();
const now = this.timeService.date;
now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0);
const redisPipeline = this.redisClient.pipeline();
@ -217,7 +218,7 @@ export class HashtagService {
@bindThis
public async getCharts(hashtags: string[], range: number): Promise<Record<string, number[]>> {
const now = new Date();
const now = this.timeService.date;
now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0);
const redisPipeline = this.redisClient.pipeline();

11
packages/backend/src/core/HttpRequestService.ts
View file

@ -19,6 +19,7 @@ import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/val
import type { IObject, IObjectWithId } from '@/core/activitypub/type.js';
import { UtilityService } from '@/core/UtilityService.js';
import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
import { TimeService } from '@/global/TimeService.js';
import type { Response } from 'node-fetch';
import type { Socket } from 'node:net';
@ -72,7 +73,7 @@ export function validateSocketConnect(allowedPrivateNetworks: PrivateNetwork[] |
declare module 'node:http' {
interface Agent {
createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket;
createConnection(options: net.NetConnectOpts, callback?: (err: Error | null, stream: net.Socket) => void): net.Socket;
}
}
@ -85,7 +86,7 @@ class HttpRequestServiceAgent extends http.Agent {
}
@bindThis
public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket {
public createConnection(options: net.NetConnectOpts, callback?: (err: Error | null, stream: net.Socket) => void): net.Socket {
const socket = super.createConnection(options, callback)
.on('connect', () => {
if (process.env.NODE_ENV === 'production') {
@ -105,7 +106,7 @@ class HttpsRequestServiceAgent extends https.Agent {
}
@bindThis
public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket {
public createConnection(options: net.NetConnectOpts, callback?: (err: Error | null, stream: net.Socket) => void): net.Socket {
const socket = super.createConnection(options, callback)
.on('connect', () => {
if (process.env.NODE_ENV === 'production') {
@ -156,8 +157,10 @@ export class HttpRequestService {
constructor(
@Inject(DI.config)
private config: Config,
private readonly apUtilityService: ApUtilityService,
private readonly utilityService: UtilityService,
private readonly timeService: TimeService,
) {
const cache = new CacheableLookup({
maxTtl: 3600, // 1hours
@ -343,7 +346,7 @@ export class HttpRequestService {
this.utilityService.assertUrl(parsedUrl, allowHttp);
const controller = new AbortController();
setTimeout(() => {
this.timeService.startTimer(() => {
controller.abort();
}, timeout);

4
packages/backend/src/core/IdService.ts
View file

@ -7,6 +7,7 @@ import { Inject, Injectable } from '@nestjs/common';
import { ulid } from 'ulid';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { TimeService } from '@/global/TimeService.js';
import { genAid, isSafeAidT, parseAid, parseAidFull } from '@/misc/id/aid.js';
import { genAidx, isSafeAidxT, parseAidx, parseAidxFull } from '@/misc/id/aidx.js';
import { genMeid, isSafeMeidT, parseMeid, parseMeidFull } from '@/misc/id/meid.js';
@ -22,6 +23,7 @@ export class IdService {
constructor(
@Inject(DI.config)
private config: Config,
private readonly timeService: TimeService,
) {
this.method = config.id.toLowerCase();
}
@ -45,7 +47,7 @@ export class IdService {
*/
@bindThis
public gen(time?: number): string {
const t = (!time || (time > Date.now())) ? Date.now() : time;
const t = (!time || (time > this.timeService.now)) ? this.timeService.now : time;
switch (this.method) {
case 'aid': return genAid(t);

117
packages/backend/src/core/InstanceStatsService.ts Normal file
View file

@ -0,0 +1,117 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, MoreThan } from 'typeorm';
import { CacheManagementService, type ManagedMemorySingleCache } from '@/global/CacheManagementService.js';
import NotesChart from '@/core/chart/charts/notes.js';
import UsersChart from '@/core/chart/charts/users.js';
import { DI } from '@/di-symbols.js';
import type { UsersRepository } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { TimeService } from '@/global/TimeService.js';
export interface InstanceStats {
/**
* The number of local posts on the instance.
* Updated hourly.
*/
notesTotal: number;
/**
* The number of local users currently registered on the instance.
* Updated hourly.
*/
usersTotal: number;
/**
* The number of local users who have been active within the past month.
* Updated daily.
*/
usersActiveMonth: number;
/**
* The number of local users who have been active within the past 6 months.
* Updated weekly.
*/
usersActiveSixMonths: number;
}
@Injectable()
export class InstanceStatsService {
private readonly activeSixMonthsCache: ManagedMemorySingleCache<number>;
private readonly activeMonthCache: ManagedMemorySingleCache<number>;
private readonly localUsersCache: ManagedMemorySingleCache<number>;
private readonly localPostsCache: ManagedMemorySingleCache<number>;
constructor(
@Inject(DI.usersRepository)
private readonly usersRepository: UsersRepository,
private readonly notesChart: NotesChart,
private readonly usersChart: UsersChart,
private readonly timeService: TimeService,
cacheManagementService: CacheManagementService,
) {
this.localPostsCache = cacheManagementService.createMemorySingleCache<number>('localPosts', 1000 * 60 * 60); // 1h
this.localUsersCache = cacheManagementService.createMemorySingleCache<number>('localUsers', 1000 * 60 * 60); // 1h
this.activeMonthCache = cacheManagementService.createMemorySingleCache<number>('activeMonth', 1000 * 60 * 60 * 24); // 1d
this.activeSixMonthsCache = cacheManagementService.createMemorySingleCache<number>('activeSixMonths', 1000 * 60 * 60 * 24 * 7); // 1w
}
@bindThis
public async fetch(): Promise<InstanceStats> {
const [notesTotal, usersTotal, usersActiveMonth, usersActiveSixMonths] = await Promise.all([
this.fetchLocalPosts(),
this.fetchLocalUsers(),
this.fetchActiveMonth(),
this.fetchActiveSixMonths(),
]);
return { notesTotal, usersTotal, usersActiveMonth, usersActiveSixMonths };
}
@bindThis
private async fetchActiveSixMonths(): Promise<number> {
return await this.activeSixMonthsCache.fetch(async () => {
const now = this.timeService.now;
const halfYearAgo = new Date(now - 15552000000);
return await this.usersRepository.countBy({
host: IsNull(),
isBot: false,
lastActiveDate: MoreThan(halfYearAgo),
});
});
}
@bindThis
private async fetchActiveMonth(): Promise<number> {
return await this.activeMonthCache.fetch(async () => {
const now = this.timeService.now;
const halfYearAgo = new Date(now - 2592000000);
return await this.usersRepository.countBy({
host: IsNull(),
isBot: false,
lastActiveDate: MoreThan(halfYearAgo),
});
});
}
@bindThis
private async fetchLocalUsers(): Promise<number> {
return await this.localUsersCache.fetch(async () => {
const chart = await this.usersChart.getChart('hour', 1, null);
return chart.local.total[0];
});
}
@bindThis
private async fetchLocalPosts(): Promise<number> {
return await this.localPostsCache.fetch(async () => {
const chart = await this.notesChart.getChart('hour', 1, null);
return chart.local.total[0];
});
}
}

16
packages/backend/src/core/LoggerService.ts
View file

@ -5,23 +5,25 @@
import { Inject, Injectable } from '@nestjs/common';
import Logger from '@/logger.js';
import { TimeService } from '@/global/TimeService.js';
import { EnvService } from '@/global/EnvService.js';
import { bindThis } from '@/decorators.js';
import type { KEYWORD } from 'color-convert/conversions.js';
import { envOption } from '@/env.js';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { KEYWORD } from 'color-convert/conversions.js';
@Injectable()
export class LoggerService {
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.console)
protected readonly console: Console,
protected readonly timeService: TimeService,
protected readonly envService: EnvService,
) {
}
@bindThis
public getLogger(domain: string, color?: KEYWORD | undefined) {
const verbose = this.config.logging?.verbose || envOption.verbose;
return new Logger(domain, color, verbose);
return new Logger(domain, color, this.envService, this.timeService, this.console);
}
}

10
packages/backend/src/core/MetaService.ts
View file

@ -12,6 +12,7 @@ import { GlobalEventService } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { TimeService, type TimerHandle } from '@/global/TimeService.js';
import { MiInstance } from '@/models/Instance.js';
import { diffArrays } from '@/misc/diff-arrays.js';
import type { MetasRepository } from '@/models/_.js';
@ -20,7 +21,7 @@ import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
export class MetaService implements OnApplicationShutdown {
private cache: MiMeta | undefined;
private intervalId: NodeJS.Timeout;
private intervalId: TimerHandle;
constructor(
@Inject(DI.redisForSub)
@ -34,16 +35,17 @@ export class MetaService implements OnApplicationShutdown {
private featuredService: FeaturedService,
private globalEventService: GlobalEventService,
private readonly timeService: TimeService,
) {
//this.onMessage = this.onMessage.bind(this);
if (process.env.NODE_ENV !== 'test') {
this.intervalId = setInterval(() => {
this.intervalId = this.timeService.startTimer(() => {
this.fetch(true).then(meta => {
// fetch内でもセットしてるけど仕様変更の可能性もあるため一応
this.cache = meta;
});
}, 1000 * 60 * 5);
}, 1000 * 60 * 5, { repeated: true });
}
this.redisForSub.on('message', this.onMessage);
@ -161,7 +163,7 @@ export class MetaService implements OnApplicationShutdown {
@bindThis
public dispose(): void {
clearInterval(this.intervalId);
this.timeService.stopTimer(this.intervalId);
this.redisForSub.off('message', this.onMessage);
}

2
packages/backend/src/core/MfmService.ts
View file

@ -299,7 +299,7 @@ export class MfmService {
(note that the `rp` are to be ignored, they only exist
for browsers who don't understand ruby)
*/
let nonRtNodes = [];
let nonRtNodes: ChildNode[] = [];
// scan children, ignore `rp`, split on `rt`
for (const child of node.childNodes) {
if (isText(child)) {

97
packages/backend/src/core/NoteCreateService.ts
View file

@ -19,6 +19,7 @@ import type { MiApp } from '@/models/App.js';
import { concat } from '@/misc/prelude/array.js';
import { IdService } from '@/core/IdService.js';
import type { MiUser, MiLocalUser, MiRemoteUser } from '@/models/User.js';
import { isLocalUser, isRemoteUser } from '@/models/User.js';
import type { IPoll } from '@/models/Poll.js';
import { MiPoll } from '@/models/Poll.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
@ -39,7 +40,6 @@ import { HashtagService } from '@/core/HashtagService.js';
import { AntennaService } from '@/core/AntennaService.js';
import { QueueService } from '@/core/QueueService.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js';
import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
@ -57,8 +57,8 @@ import { IdentifiableError } from '@/misc/identifiable-error.js';
import { LatestNoteService } from '@/core/LatestNoteService.js';
import { CollapsedQueue } from '@/misc/collapsed-queue.js';
import { CacheService } from '@/core/CacheService.js';
import { TimeService } from '@/global/TimeService.js';
import { NoteVisibilityService } from '@/core/NoteVisibilityService.js';
import { isPureRenote } from '@/misc/is-renote.js';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -199,7 +199,6 @@ export class NoteCreateService implements OnApplicationShutdown {
@Inject(DI.channelFollowingsRepository)
private channelFollowingsRepository: ChannelFollowingsRepository,
private userEntityService: UserEntityService,
private noteEntityService: NoteEntityService,
private idService: IdService,
private globalEventService: GlobalEventService,
@ -225,9 +224,10 @@ export class NoteCreateService implements OnApplicationShutdown {
private userBlockingService: UserBlockingService,
private cacheService: CacheService,
private latestNoteService: LatestNoteService,
private readonly timeService: TimeService,
private readonly noteVisibilityService: NoteVisibilityService,
) {
this.updateNotesCountQueue = new CollapsedQueue(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseNotesCount, this.performUpdateNotesCount);
this.updateNotesCountQueue = new CollapsedQueue(this.timeService, process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseNotesCount, this.performUpdateNotesCount);
}
@bindThis
@ -254,7 +254,7 @@ export class NoteCreateService implements OnApplicationShutdown {
data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId });
}
if (data.createdAt == null) data.createdAt = new Date();
if (data.createdAt == null) data.createdAt = this.timeService.date;
if (data.visibility == null) data.visibility = 'public';
if (data.localOnly == null) data.localOnly = false;
if (data.channel != null) data.visibility = 'public';
@ -525,7 +525,7 @@ export class NoteCreateService implements OnApplicationShutdown {
if (mentionedUsers.length > 0) {
insert.mentions = mentionedUsers.map(u => u.id);
const profiles = await this.userProfilesRepository.findBy({ userId: In(insert.mentions) });
insert.mentionedRemoteUsers = JSON.stringify(mentionedUsers.filter(u => this.userEntityService.isRemoteUser(u)).map(u => {
insert.mentionedRemoteUsers = JSON.stringify(mentionedUsers.filter(u => isRemoteUser(u)).map(u => {
const profile = profiles.find(p => p.userId === u.id);
const url = profile != null ? profile.url : null;
return {
@ -591,7 +591,7 @@ export class NoteCreateService implements OnApplicationShutdown {
// Register host
if (this.meta.enableStatsForFederatedInstances) {
if (this.userEntityService.isRemoteUser(user)) {
if (isRemoteUser(user)) {
this.federatedInstanceService.fetchOrRegister(user.host).then(async i => {
if (!this.isRenote(note) || this.isQuote(note)) {
this.updateNotesCountQueue.enqueue(i.id, 1);
@ -614,7 +614,7 @@ export class NoteCreateService implements OnApplicationShutdown {
// Increment notes count (user)
this.incNotesCountOfUser(user);
} else {
this.usersRepository.update({ id: user.id }, { updatedAt: new Date() });
this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date });
}
this.pushToTl(note, user);
@ -658,11 +658,11 @@ export class NoteCreateService implements OnApplicationShutdown {
}
if (data.poll && data.poll.expiresAt) {
const delay = data.poll.expiresAt.getTime() - Date.now();
const delay = data.poll.expiresAt.getTime() - this.timeService.now;
this.queueService.endedPollNotificationQueue.add(note.id, {
noteId: note.id,
}, {
jobId: `pollEnd:${note.id}`,
jobId: `pollEnd_${note.id}`,
delay,
removeOnComplete: {
age: 3600 * 24 * 7, // keep up to 7 days
@ -676,7 +676,7 @@ export class NoteCreateService implements OnApplicationShutdown {
}
if (!silent) {
if (this.userEntityService.isLocalUser(user)) this.activeUsersChart.write(user);
if (isLocalUser(user)) this.activeUsersChart.write(user);
// Pack the note
const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true });
@ -752,26 +752,26 @@ export class NoteCreateService implements OnApplicationShutdown {
nm.notify();
//#region AP deliver
if (!data.localOnly && this.userEntityService.isLocalUser(user)) {
if (!data.localOnly && isLocalUser(user)) {
trackTask(async () => {
const noteActivity = await this.apRendererService.renderNoteOrRenoteActivity(note, user, { renote: data.renote });
const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity);
// メンションされたリモートユーザーに配送
for (const u of mentionedUsers.filter(u => this.userEntityService.isRemoteUser(u))) {
for (const u of mentionedUsers.filter(u => isRemoteUser(u))) {
dm.addDirectRecipe(u as MiRemoteUser);
}
// 投稿がリプライかつ投稿者がローカルユーザーかつリプライ先の投稿の投稿者がリモートユーザーなら配送
if (data.reply && data.reply.userHost !== null) {
const u = await this.usersRepository.findOneBy({ id: data.reply.userId });
if (u && this.userEntityService.isRemoteUser(u)) dm.addDirectRecipe(u);
if (u && isRemoteUser(u)) dm.addDirectRecipe(u);
}
// 投稿がRenoteかつ投稿者がローカルユーザーかつRenote元の投稿の投稿者がリモートユーザーなら配送
if (data.renote && data.renote.userHost !== null) {
const u = await this.usersRepository.findOneBy({ id: data.renote.userId });
if (u && this.userEntityService.isRemoteUser(u)) dm.addDirectRecipe(u);
if (u && isRemoteUser(u)) dm.addDirectRecipe(u);
}
// フォロワーに配送
@ -792,7 +792,7 @@ export class NoteCreateService implements OnApplicationShutdown {
if (data.channel) {
this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1);
this.channelsRepository.update(data.channel.id, {
lastNotedAt: new Date(),
lastNotedAt: this.timeService.date,
});
this.notesRepository.countBy({
@ -814,27 +814,20 @@ export class NoteCreateService implements OnApplicationShutdown {
if (!user.noindex) this.index(note);
}
@bindThis
public isPureRenote(note: Option): note is PureRenoteOption {
return this.isRenote(note) && !this.isQuote(note);
}
/**
* @deprecated Use the exported function instead
*/
readonly isPureRenote = isPureRenote;
@bindThis
private isRenote(note: Option): note is Option & { renote: MiNote } {
return note.renote != null;
}
/**
* @deprecated Use the exported function instead
*/
readonly isRenote = isRenote;
@bindThis
private isQuote(note: Option & { renote: MiNote }): note is Option & { renote: MiNote } & (
{ text: string } | { cw: string } | { reply: MiNote } | { poll: IPoll } | { files: MiDriveFile[] }
) {
// NOTE: SYNC WITH misc/is-quote.ts
return note.text != null ||
note.reply != null ||
note.cw != null ||
note.poll != null ||
(note.files != null && note.files.length > 0);
}
/**
* @deprecated Use the exported function instead
*/
readonly isQuote = isQuote;
@bindThis
private async incRenoteCount(renote: MiNote, user: MiUser) {
@ -846,7 +839,7 @@ export class NoteCreateService implements OnApplicationShutdown {
.execute();
// 30%の確率、3日以内に投稿されたノートの場合ハイライト用ランキング更新
if (user.isExplorable && Math.random() < 0.3 && (Date.now() - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) {
if (user.isExplorable && Math.random() < 0.3 && (this.timeService.now - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) {
const policies = await this.roleService.getUserPolicies(user);
if (policies.canTrend) {
if (renote.channelId != null) {
@ -874,7 +867,7 @@ export class NoteCreateService implements OnApplicationShutdown {
]);
// Only create mention events for local users, and users for whom the note is visible
for (const u of mentionedUsers.filter(u => (note.visibility !== 'specified' || note.visibleUserIds.some(x => x === u.id)) && this.userEntityService.isLocalUser(u))) {
for (const u of mentionedUsers.filter(u => (note.visibility !== 'specified' || note.visibleUserIds.some(x => x === u.id)) && isLocalUser(u))) {
const threadId = note.threadId ?? note.id;
const isThreadMuted = threadMutings.get(u.id)?.has(threadId);
@ -913,7 +906,7 @@ export class NoteCreateService implements OnApplicationShutdown {
private incNotesCountOfUser(user: { id: MiUser['id']; }) {
this.usersRepository.createQueryBuilder().update()
.set({
updatedAt: new Date(),
updatedAt: this.timeService.date,
notesCount: () => '"notesCount" + 1',
})
.where('id = :id', { id: user.id })
@ -966,12 +959,7 @@ export class NoteCreateService implements OnApplicationShutdown {
// eslint-disable-next-line prefer-const
let [followings, userListMemberships] = await Promise.all([
this.cacheService.getNonHibernatedFollowers(user.id),
this.userListMembershipsRepository.find({
where: {
userId: user.id,
},
select: ['userListId', 'userListUserId', 'withReplies'],
}),
this.cacheService.userListMembershipsCache.fetch(user.id).then(ms => ms.values().toArray()),
]);
if (note.visibility === 'followers') {
@ -1077,7 +1065,7 @@ export class NoteCreateService implements OnApplicationShutdown {
const hibernatedUsers = await this.usersRepository.find({
where: {
id: In(samples.map(x => x.followerId)),
lastActiveDate: LessThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 50))),
lastActiveDate: LessThan(new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 50))),
},
select: ['id'],
});
@ -1162,3 +1150,22 @@ export class NoteCreateService implements OnApplicationShutdown {
}
}
}
export function isPureRenote(note: Option): note is PureRenoteOption {
return isRenote(note) && !isQuote(note);
}
export function isRenote(note: Option): note is Option & { renote: MiNote } {
return note.renote != null;
}
export function isQuote(note: Option & { renote: MiNote }): note is Option & { renote: MiNote } & (
{ text: string } | { cw: string } | { reply: MiNote } | { poll: IPoll } | { files: MiDriveFile[] }
) {
// NOTE: SYNC WITH misc/is-quote.ts
return note.text != null ||
note.reply != null ||
note.cw != null ||
note.poll != null ||
(note.files != null && note.files.length > 0);
}

22
packages/backend/src/core/NoteDeleteService.ts
View file

@ -6,6 +6,7 @@
import { Brackets, In, IsNull, Not } from 'typeorm';
import { Injectable, Inject } from '@nestjs/common';
import type { MiUser, MiLocalUser, MiRemoteUser } from '@/models/User.js';
import { isLocalUser, isRemoteUser } from '@/models/User.js';
import { MiNote, IMentionedRemoteUsers } from '@/models/Note.js';
import type { InstancesRepository, MiMeta, NotesRepository, UsersRepository } from '@/models/_.js';
import { RelayService } from '@/core/RelayService.js';
@ -18,15 +19,15 @@ import InstanceChart from '@/core/chart/charts/instance.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import { SearchService } from '@/core/SearchService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { isQuote, isRenote } from '@/misc/is-renote.js';
import { LatestNoteService } from '@/core/LatestNoteService.js';
import { ApLogService } from '@/core/ApLogService.js';
import Logger from '@/logger.js';
import { LoggerService } from './LoggerService.js';
import type Logger from '@/logger.js';
import { TimeService } from '@/global/TimeService.js';
import { LoggerService } from '@/core/LoggerService.js';
@Injectable()
export class NoteDeleteService {
@ -48,7 +49,6 @@ export class NoteDeleteService {
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
private userEntityService: UserEntityService,
private globalEventService: GlobalEventService,
private relayService: RelayService,
private federatedInstanceService: FederatedInstanceService,
@ -61,6 +61,8 @@ export class NoteDeleteService {
private instanceChart: InstanceChart,
private latestNoteService: LatestNoteService,
private readonly apLogService: ApLogService,
private readonly timeService: TimeService,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('note-delete-service');
@ -72,7 +74,7 @@ export class NoteDeleteService {
* @param note 投稿
*/
async delete(user: { id: MiUser['id']; uri: MiUser['uri']; host: MiUser['host']; isBot: MiUser['isBot']; }, note: MiNote, quiet = false, deleter?: MiUser) {
const deletedAt = new Date();
const deletedAt = this.timeService.date;
const cascadingNotes = await this.findCascadingNotes(note);
if (note.replyId) {
@ -92,7 +94,7 @@ export class NoteDeleteService {
});
//#region ローカルの投稿なら削除アクティビティを配送
if (this.userEntityService.isLocalUser(user) && !note.localOnly) {
if (isLocalUser(user) && !note.localOnly) {
let renote: MiNote | null = null;
// if deleted note is renote
@ -113,7 +115,7 @@ export class NoteDeleteService {
const federatedLocalCascadingNotes = (cascadingNotes).filter(note => !note.localOnly && note.userHost == null); // filter out local-only notes
for (const cascadingNote of federatedLocalCascadingNotes) {
if (!cascadingNote.user) continue;
if (!this.userEntityService.isLocalUser(cascadingNote.user)) continue;
if (!isLocalUser(cascadingNote.user)) continue;
const content = this.apRendererService.addContext(this.apRendererService.renderDelete(this.apRendererService.renderTombstone(`${this.config.url}/notes/${cascadingNote.id}`), cascadingNote.user));
this.deliverToConcerned(cascadingNote.user, cascadingNote, content);
}
@ -128,11 +130,11 @@ export class NoteDeleteService {
// Decrement notes count (user)
this.decNotesCountOfUser(user);
} else {
this.usersRepository.update({ id: user.id }, { updatedAt: new Date() });
this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date });
}
if (this.meta.enableStatsForFederatedInstances) {
if (this.userEntityService.isRemoteUser(user)) {
if (isRemoteUser(user)) {
this.federatedInstanceService.fetchOrRegister(user.host).then(async i => {
if (note.renoteId && note.text || !note.renoteId) {
this.instancesRepository.decrement({ id: i.id }, 'notesCount', 1);
@ -180,7 +182,7 @@ export class NoteDeleteService {
private decNotesCountOfUser(user: { id: MiUser['id']; }) {
this.usersRepository.createQueryBuilder().update()
.set({
updatedAt: new Date(),
updatedAt: this.timeService.date,
notesCount: () => '"notesCount" - 1',
})
.where('id = :id', { id: user.id })

44
packages/backend/src/core/NoteEditService.ts
View file

@ -20,6 +20,7 @@ import type { MiApp } from '@/models/App.js';
import { concat } from '@/misc/prelude/array.js';
import { IdService } from '@/core/IdService.js';
import type { MiUser, MiLocalUser, MiRemoteUser } from '@/models/User.js';
import { isRemoteUser, isLocalUser } from '@/models/User.js';
import { MiPoll, type IPoll } from '@/models/Poll.js';
import type { MiChannel } from '@/models/Channel.js';
import { normalizeForSearch } from '@/misc/normalize-for-search.js';
@ -34,7 +35,6 @@ import { NotificationService } from '@/core/NotificationService.js';
import { UserWebhookService } from '@/core/UserWebhookService.js';
import { QueueService } from '@/core/QueueService.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js';
import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
@ -52,6 +52,7 @@ import { IdentifiableError } from '@/misc/identifiable-error.js';
import { LatestNoteService } from '@/core/LatestNoteService.js';
import { CollapsedQueue } from '@/misc/collapsed-queue.js';
import { NoteCreateService } from '@/core/NoteCreateService.js';
import { TimeService } from '@/global/TimeService.js';
import { NoteVisibilityService } from '@/core/NoteVisibilityService.js';
import { isPureRenote } from '@/misc/is-renote.js';
@ -200,7 +201,6 @@ export class NoteEditService implements OnApplicationShutdown {
@Inject(DI.pollsRepository)
private pollsRepository: PollsRepository,
private userEntityService: UserEntityService,
private noteEntityService: NoteEntityService,
private idService: IdService,
private globalEventService: GlobalEventService,
@ -222,9 +222,10 @@ export class NoteEditService implements OnApplicationShutdown {
private cacheService: CacheService,
private latestNoteService: LatestNoteService,
private noteCreateService: NoteCreateService,
private readonly timeService: TimeService,
private readonly noteVisibilityService: NoteVisibilityService,
) {
this.updateNotesCountQueue = new CollapsedQueue(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseNotesCount, this.performUpdateNotesCount);
this.updateNotesCountQueue = new CollapsedQueue(this.timeService, process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseNotesCount, this.performUpdateNotesCount);
}
@bindThis
@ -273,7 +274,7 @@ export class NoteEditService implements OnApplicationShutdown {
data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId });
}
if (data.updatedAt == null) data.updatedAt = new Date();
if (data.updatedAt == null) data.updatedAt = this.timeService.date;
if (data.visibility == null) data.visibility = 'public';
if (data.localOnly == null) data.localOnly = false;
if (data.channel != null) data.visibility = 'public';
@ -494,12 +495,12 @@ export class NoteEditService implements OnApplicationShutdown {
cw: update.cw || undefined,
fileIds: undefined,
oldDate: exists ? oldnote.updatedAt as Date : this.idService.parse(oldnote.id).date,
updatedAt: new Date(),
updatedAt: this.timeService.date,
});
const note = new MiNote({
id: oldnote.id,
updatedAt: data.updatedAt ? data.updatedAt : new Date(),
updatedAt: data.updatedAt ? data.updatedAt : this.timeService.date,
fileIds: data.files ? data.files.map(file => file.id) : [],
replyId: oldnote.replyId,
renoteId: data.renote ? data.renote.id : null,
@ -545,7 +546,7 @@ export class NoteEditService implements OnApplicationShutdown {
if (mentionedUsers.length > 0) {
note.mentions = mentionedUsers.map(u => u.id);
const profiles = await this.userProfilesRepository.findBy({ userId: In(note.mentions) });
note.mentionedRemoteUsers = JSON.stringify(mentionedUsers.filter(u => this.userEntityService.isRemoteUser(u)).map(u => {
note.mentionedRemoteUsers = JSON.stringify(mentionedUsers.filter(u => isRemoteUser(u)).map(u => {
const profile = profiles.find(p => p.userId === u.id);
const url = profile != null ? profile.url : null;
return {
@ -608,7 +609,7 @@ export class NoteEditService implements OnApplicationShutdown {
}, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) {
// Register host
if (this.meta.enableStatsForFederatedInstances) {
if (this.userEntityService.isRemoteUser(user)) {
if (isRemoteUser(user)) {
this.federatedInstanceService.fetchOrRegister(user.host).then(async i => {
if (note.renote && note.text || !note.renote) {
this.updateNotesCountQueue.enqueue(i.id, 1);
@ -620,25 +621,25 @@ export class NoteEditService implements OnApplicationShutdown {
}
}
this.usersRepository.update({ id: user.id }, { updatedAt: new Date() });
this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date });
// ハッシュタグ更新
this.pushToTl(note, user);
if (data.poll && data.poll.expiresAt) {
const delay = data.poll.expiresAt.getTime() - Date.now();
const delay = data.poll.expiresAt.getTime() - this.timeService.now;
this.queueService.endedPollNotificationQueue.remove(`pollEnd:${note.id}`);
this.queueService.endedPollNotificationQueue.add(note.id, {
noteId: note.id,
}, {
jobId: `pollEnd:${note.id}`,
jobId: `pollEnd_${note.id}`,
delay,
removeOnComplete: true,
});
}
if (!silent) {
if (this.userEntityService.isLocalUser(user)) this.activeUsersChart.write(user);
if (isLocalUser(user)) this.activeUsersChart.write(user);
// Pack the note
const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true });
@ -680,26 +681,26 @@ export class NoteEditService implements OnApplicationShutdown {
nm.notify();
//#region AP deliver
if (!data.localOnly && this.userEntityService.isLocalUser(user)) {
if (!data.localOnly && isLocalUser(user)) {
trackTask(async () => {
const noteActivity = await this.apRendererService.renderNoteOrRenoteActivity(note, user, { renote: data.renote });
const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity);
// メンションされたリモートユーザーに配送
for (const u of mentionedUsers.filter(u => this.userEntityService.isRemoteUser(u))) {
for (const u of mentionedUsers.filter(u => isRemoteUser(u))) {
dm.addDirectRecipe(u as MiRemoteUser);
}
// 投稿がリプライかつ投稿者がローカルユーザーかつリプライ先の投稿の投稿者がリモートユーザーなら配送
if (data.reply && data.reply.userHost !== null) {
const u = await this.usersRepository.findOneBy({ id: data.reply.userId });
if (u && this.userEntityService.isRemoteUser(u)) dm.addDirectRecipe(u);
if (u && isRemoteUser(u)) dm.addDirectRecipe(u);
}
// 投稿がRenoteかつ投稿者がローカルユーザーかつRenote元の投稿の投稿者がリモートユーザーなら配送
if (this.isRenote(data) && data.renote.userHost !== null) {
const u = await this.usersRepository.findOneBy({ id: data.renote.userId });
if (u && this.userEntityService.isRemoteUser(u)) dm.addDirectRecipe(u);
if (u && isRemoteUser(u)) dm.addDirectRecipe(u);
}
// フォロワーに配送
@ -738,7 +739,7 @@ export class NoteEditService implements OnApplicationShutdown {
if (data.channel) {
this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1);
this.channelsRepository.update(data.channel.id, {
lastNotedAt: new Date(),
lastNotedAt: this.timeService.date,
});
this.notesRepository.countBy({
@ -830,12 +831,7 @@ export class NoteEditService implements OnApplicationShutdown {
// eslint-disable-next-line prefer-const
let [followings, userListMemberships] = await Promise.all([
this.cacheService.getNonHibernatedFollowers(user.id),
this.userListMembershipsRepository.find({
where: {
userId: user.id,
},
select: ['userListId', 'userListUserId', 'withReplies'],
}),
this.cacheService.userListMembershipsCache.fetch(user.id).then(ms => ms.values().toArray()),
]);
if (note.visibility === 'followers') {
@ -941,7 +937,7 @@ export class NoteEditService implements OnApplicationShutdown {
const hibernatedUsers = await this.usersRepository.find({
where: {
id: In(samples.map(x => x.followerId)),
lastActiveDate: LessThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 50))),
lastActiveDate: LessThan(new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 50))),
},
select: ['id'],
});

36
packages/backend/src/core/NoteVisibilityService.ts
View file

@ -4,15 +4,19 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { CacheService } from '@/core/CacheService.js';
import type { MiNote } from '@/models/Note.js';
import type { MiUser } from '@/models/User.js';
import { bindThis } from '@/decorators.js';
import type { MiFollowing } from '@/models/Following.js';
import type { MiInstance } from '@/models/Instance.js';
import type { MiUserListMembership } from '@/models/UserListMembership.js';
import type { NotesRepository } from '@/models/_.js';
import type { Packed } from '@/misc/json-schema.js';
import { IdService } from '@/core/IdService.js';
import { awaitAll } from '@/misc/prelude/await-all.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import type { MiFollowing, MiInstance, NotesRepository } from '@/models/_.js';
import { CacheService } from '@/core/CacheService.js';
import { IdService } from '@/core/IdService.js';
import { TimeService } from '@/global/TimeService.js';
import { bindThis } from '@/decorators.js';
import { awaitAll } from '@/misc/prelude/await-all.js';
import { DI } from '@/di-symbols.js';
/**
@ -50,6 +54,12 @@ export interface NoteVisibilityFilters {
* If false (default), then silence is enforced for all notes.
*/
includeSilencedAuthor?: boolean;
/**
* Set to an ID to apply visibility from the context of a specific user list.
* Membership and "with replies" settings will be adopted from this list.
*/
listContext?: string | null;
}
@Injectable()
@ -61,6 +71,7 @@ export class NoteVisibilityService {
private readonly cacheService: CacheService,
private readonly idService: IdService,
private readonly federatedInstanceService: FederatedInstanceService,
private readonly timeService: TimeService,
) {}
@bindThis
@ -151,7 +162,7 @@ export class NoteVisibilityService {
}
@bindThis
public async populateData(user: PopulatedMe, hint?: Partial<NoteVisibilityData>): Promise<NoteVisibilityData> {
public async populateData(user: PopulatedMe, hint?: Partial<NoteVisibilityData>, filters?: NoteVisibilityFilters): Promise<NoteVisibilityData> {
// noinspection ES6MissingAwait
const [
userBlockers,
@ -161,6 +172,7 @@ export class NoteVisibilityService {
userMutedUsers,
userMutedUserRenotes,
userMutedInstances,
userListMemberships,
] = await Promise.all([
user ? (hint?.userBlockers ?? this.cacheService.userBlockedCache.fetch(user.id)) : null,
user ? (hint?.userFollowings ?? this.cacheService.userFollowingsCache.fetch(user.id)) : null,
@ -169,6 +181,7 @@ export class NoteVisibilityService {
user ? (hint?.userMutedUsers ?? this.cacheService.userMutingsCache.fetch(user.id)) : null,
user ? (hint?.userMutedUserRenotes ?? this.cacheService.renoteMutingsCache.fetch(user.id)) : null,
user ? (hint?.userMutedInstances ?? this.cacheService.userProfileCache.fetch(user.id).then(p => new Set(p.mutedInstances))) : null,
filters?.listContext ? (hint?.userListMemberships ?? this.cacheService.listUserMembershipsCache.fetch(filters.listContext)) : null,
]);
return {
@ -179,6 +192,7 @@ export class NoteVisibilityService {
userMutedUsers,
userMutedUserRenotes,
userMutedInstances,
userListMemberships,
};
}
@ -281,7 +295,7 @@ export class NoteVisibilityService {
const createdAt = new Date(note.createdAt).valueOf();
// I don't understand this logic, but I tried to break it out for readability
const followersOnlyOpt1 = followersOnlyBefore <= 0 && (Date.now() - createdAt > 0 - followersOnlyBefore);
const followersOnlyOpt1 = followersOnlyBefore <= 0 && (this.timeService.now - createdAt > 0 - followersOnlyBefore);
const followersOnlyOpt2 = followersOnlyBefore > 0 && (createdAt < followersOnlyBefore);
if (followersOnlyOpt1 || followersOnlyOpt2) {
note.visibility = 'followers';
@ -311,7 +325,7 @@ export class NoteVisibilityService {
const createdAt = note.createdAt.valueOf();
// I don't understand this logic, but I tried to break it out for readability
const hiddenOpt1 = hiddenBefore <= 0 && (Date.now() - createdAt > 0 - hiddenBefore);
const hiddenOpt1 = hiddenBefore <= 0 && (this.timeService.now - createdAt > 0 - hiddenBefore);
const hiddenOpt2 = hiddenBefore > 0 && (createdAt < hiddenBefore);
if (hiddenOpt1 || hiddenOpt2) return true;
}
@ -396,6 +410,9 @@ export class NoteVisibilityService {
// Don't silence if we follow w/ replies
if (user && data.userFollowings?.get(user.id)?.withReplies) return false;
// Don't silence if we're viewing in a list with replies
if (data.userListMemberships?.get(note.userId)?.withReplies) return false;
// Silence otherwise
return true;
}
@ -409,6 +426,9 @@ export interface NoteVisibilityData extends NotePopulationData {
userMutedUsers: Set<string> | null;
userMutedUserRenotes: Set<string> | null;
userMutedInstances: Set<string> | null;
// userId => membership (already scoped to listContext)
userListMemberships: Map<string, MiUserListMembership> | null;
}
export interface NotePopulationData {

19
packages/backend/src/core/NotificationService.ts
View file

@ -22,6 +22,7 @@ import type { Config } from '@/config.js';
import { UserListService } from '@/core/UserListService.js';
import { FilterUnionByProperty, groupedNotificationTypes, obsoleteNotificationTypes } from '@/types.js';
import { trackPromise } from '@/misc/promise-tracker.js';
import { TimeService } from '@/global/TimeService.js';
@Injectable()
export class NotificationService implements OnApplicationShutdown {
@ -43,6 +44,7 @@ export class NotificationService implements OnApplicationShutdown {
private pushNotificationService: PushNotificationService,
private cacheService: CacheService,
private userListService: UserListService,
private readonly timeService: TimeService,
) {
}
@ -93,7 +95,10 @@ export class NotificationService implements OnApplicationShutdown {
data: Omit<FilterUnionByProperty<MiNotification, 'type', T>, 'type' | 'id' | 'createdAt' | 'notifierId'>,
notifierId?: MiUser['id'] | null,
): Promise<MiNotification | null> {
const profile = await this.cacheService.userProfileCache.fetch(notifieeId);
const [profile, notifiee] = await Promise.all([
this.cacheService.userProfileCache.fetch(notifieeId),
this.cacheService.findUserById(notifieeId),
]);
// 古いMisskeyバージョンのキャッシュが残っている可能性がある
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
@ -139,14 +144,14 @@ export class NotificationService implements OnApplicationShutdown {
return null;
}
} else if (recieveConfig?.type === 'list') {
const isMember = await this.userListService.membersCache.fetch(recieveConfig.userListId).then(members => members.has(notifierId));
const isMember = await this.cacheService.listUserMembershipsCache.fetch(recieveConfig.userListId).then(members => members.has(notifierId));
if (!isMember) {
return null;
}
}
}
const createdAt = new Date();
const createdAt = this.timeService.date;
let notification: FilterUnionByProperty<MiNotification, 'type', T>;
let redisId: string;
@ -177,7 +182,7 @@ export class NotificationService implements OnApplicationShutdown {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
} while (true);
const packed = await this.notificationEntityService.pack(notification, notifieeId, {});
const packed = await this.notificationEntityService.pack(notification, notifiee, {});
if (packed == null) return null;
@ -187,15 +192,15 @@ export class NotificationService implements OnApplicationShutdown {
// 2秒経っても(今回作成した)通知が既読にならなかったら「未読の通知がありますよ」イベントを発行する
// テスト通知の場合は即時発行
const interval = notification.type === 'test' ? 0 : 2000;
setTimeout(interval, 'unread notification', { signal: this.#shutdownController.signal }).then(async () => {
this.timeService.startPromiseTimer(interval, 'unread notification', { signal: this.#shutdownController.signal }).then(async () => {
const latestReadNotificationId = await this.redisClient.get(`latestReadNotification:${notifieeId}`);
if (latestReadNotificationId && (latestReadNotificationId >= redisId)) return;
this.globalEventService.publishMainStream(notifieeId, 'unreadNotification', packed);
this.pushNotificationService.pushNotification(notifieeId, 'notification', packed);
if (type === 'follow') this.emailNotificationFollow(notifieeId, await this.usersRepository.findOneByOrFail({ id: notifierId! }));
if (type === 'receiveFollowRequest') this.emailNotificationReceiveFollowRequest(notifieeId, await this.usersRepository.findOneByOrFail({ id: notifierId! }));
if (type === 'follow') this.emailNotificationFollow(notifieeId, await this.cacheService.findUserById(notifierId!));
if (type === 'receiveFollowRequest') this.emailNotificationReceiveFollowRequest(notifieeId, await this.cacheService.findUserById(notifierId!));
}, () => { /* aborted, ignore it */ });
return notification;

5
packages/backend/src/core/PollService.ts
View file

@ -11,10 +11,10 @@ import { RelayService } from '@/core/RelayService.js';
import { IdService } from '@/core/IdService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js';
import { bindThis } from '@/decorators.js';
import { UserBlockingService } from '@/core/UserBlockingService.js';
import { isLocalUser } from '@/models/User.js';
@Injectable()
export class PollService {
@ -31,7 +31,6 @@ export class PollService {
@Inject(DI.pollVotesRepository)
private pollVotesRepository: PollVotesRepository,
private userEntityService: UserEntityService,
private idService: IdService,
private relayService: RelayService,
private globalEventService: GlobalEventService,
@ -96,7 +95,7 @@ export class PollService {
const user = await this.usersRepository.findOneBy({ id: note.userId });
if (user == null) throw new Error('note not found');
if (this.userEntityService.isLocalUser(user)) {
if (isLocalUser(user)) {
const content = this.apRendererService.addContext(this.apRendererService.renderUpdate(await this.apRendererService.renderNote(note, user, false), user));
await this.apDeliverManagerService.deliverToFollowers(user, content);
await this.relayService.deliverToRelays(user, content);

31
packages/backend/src/core/PushNotificationService.ts
View file

@ -12,8 +12,8 @@ import type { Packed } from '@/misc/json-schema.js';
import { getNoteSummary } from '@/misc/get-note-summary.js';
import type { MiMeta, MiSwSubscription, SwSubscriptionsRepository } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js';
import { TimeService } from '@/global/TimeService.js';
// Defined also packages/sw/types.ts#L13
type PushNotificationsTypes = {
@ -48,8 +48,8 @@ function truncateBody<T extends keyof PushNotificationsTypes>(type: T, body: Pus
}
@Injectable()
export class PushNotificationService implements OnApplicationShutdown {
private subscriptionsCache: QuantumKVCache<MiSwSubscription[]>;
export class PushNotificationService {
private readonly subscriptionsCache: ManagedQuantumKVCache<MiSwSubscription[]>;
constructor(
@Inject(DI.config)
@ -63,11 +63,16 @@ export class PushNotificationService implements OnApplicationShutdown {
@Inject(DI.swSubscriptionsRepository)
private swSubscriptionsRepository: SwSubscriptionsRepository,
private readonly internalEventService: InternalEventService,
private readonly timeService: TimeService,
cacheManagementService: CacheManagementService,
) {
this.subscriptionsCache = new QuantumKVCache<MiSwSubscription[]>(this.internalEventService, 'userSwSubscriptions', {
this.subscriptionsCache = cacheManagementService.createQuantumKVCache<MiSwSubscription[]>('userSwSubscriptions', {
lifetime: 1000 * 60 * 60 * 1, // 1h
fetcher: (key) => this.swSubscriptionsRepository.findBy({ userId: key }),
fetcher: async userId => await this.swSubscriptionsRepository.findBy({ userId }),
// optionalFetcher not needed
// bulkFetcher not needed
});
}
@ -99,7 +104,7 @@ export class PushNotificationService implements OnApplicationShutdown {
type,
body: (type === 'notification' || type === 'unreadAntennaNote') ? truncateBody(type, body) : body,
userId,
dateTime: Date.now(),
dateTime: this.timeService.now,
}), {
proxy: this.config.proxy,
}).catch((err: any) => {
@ -125,14 +130,4 @@ export class PushNotificationService implements OnApplicationShutdown {
public async refreshCache(userId: string): Promise<void> {
await this.subscriptionsCache.refresh(userId);
}
@bindThis
public dispose(): void {
this.subscriptionsCache.dispose();
}
@bindThis
public onApplicationShutdown(signal?: string | undefined): void {
this.dispose();
}
}

13
packages/backend/src/core/QueueModule.ts
View file

@ -10,6 +10,8 @@ import type { Config } from '@/config.js';
import { baseQueueOptions, QUEUE } from '@/queue/const.js';
import { allSettled } from '@/misc/promise-tracker.js';
import Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import {
DeliverJobData,
EndedPollNotificationJobData,
@ -142,7 +144,7 @@ export class QueueModule implements OnApplicationShutdown {
await allSettled();
// And then close all queues
this.logger.info('Closing BullMQ queues...');
await Promise.all([
await Promise.allSettled([
this.systemQueue.close(),
this.endedPollNotificationQueue.close(),
this.deliverQueue.close(),
@ -153,10 +155,17 @@ export class QueueModule implements OnApplicationShutdown {
this.userWebhookDeliverQueue.close(),
this.systemWebhookDeliverQueue.close(),
this.scheduleNotePostQueue.close(),
]);
]).then(res => {
for (const result of res) {
if (result.status === 'rejected') {
this.logger.error(`Error closing queue: ${renderInlineError(result.reason)}`);
}
}
});
this.logger.info('Queue module disposed.');
}
@bindThis
async onApplicationShutdown(signal: string): Promise<void> {
await this.dispose();
}

42
packages/backend/src/core/QueueService.ts
View file

@ -4,7 +4,7 @@
*/
import { randomUUID } from 'node:crypto';
import { Inject, Injectable } from '@nestjs/common';
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { MetricsTime, type JobType } from 'bullmq';
import { parse as parseRedisInfo } from 'redis-info';
import type { IActivity } from '@/core/activitypub/type.js';
@ -16,8 +16,9 @@ import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js';
import { ApRequestCreator } from '@/core/activitypub/ApRequestService.js';
import { type SystemWebhookPayload } from '@/core/SystemWebhookService.js';
import { MiNote } from '@/models/Note.js';
import { TimeService } from '@/global/TimeService.js';
import type { SystemWebhookPayload } from '@/core/SystemWebhookService.js';
import type { MiNote } from '@/models/Note.js';
import { type UserWebhookPayload } from './UserWebhookService.js';
import type {
DbJobData,
@ -56,7 +57,7 @@ export const QUEUE_TYPES = [
] as const;
@Injectable()
export class QueueService {
export class QueueService implements OnModuleInit {
constructor(
@Inject(DI.config)
private config: Config,
@ -71,57 +72,62 @@ export class QueueService {
@Inject('queue:userWebhookDeliver') public userWebhookDeliverQueue: UserWebhookDeliverQueue,
@Inject('queue:systemWebhookDeliver') public systemWebhookDeliverQueue: SystemWebhookDeliverQueue,
@Inject('queue:scheduleNotePost') public ScheduleNotePostQueue: ScheduleNotePostQueue,
) {
this.systemQueue.add('tickCharts', {
private readonly timeService: TimeService,
) {}
@bindThis
public async onModuleInit() {
await this.systemQueue.add('tickCharts', {
}, {
repeat: { pattern: '55 * * * *' },
removeOnComplete: 10,
removeOnFail: 30,
});
this.systemQueue.add('resyncCharts', {
await this.systemQueue.add('resyncCharts', {
}, {
repeat: { pattern: '0 0 * * *' },
removeOnComplete: 10,
removeOnFail: 30,
});
this.systemQueue.add('cleanCharts', {
await this.systemQueue.add('cleanCharts', {
}, {
repeat: { pattern: '0 0 * * *' },
removeOnComplete: 10,
removeOnFail: 30,
});
this.systemQueue.add('aggregateRetention', {
await this.systemQueue.add('aggregateRetention', {
}, {
repeat: { pattern: '0 0 * * *' },
removeOnComplete: 10,
removeOnFail: 30,
});
this.systemQueue.add('clean', {
await this.systemQueue.add('clean', {
}, {
repeat: { pattern: '0 0 * * *' },
removeOnComplete: 10,
removeOnFail: 30,
});
this.systemQueue.add('checkExpiredMutings', {
await this.systemQueue.add('checkExpiredMutings', {
}, {
repeat: { pattern: '*/5 * * * *' },
removeOnComplete: 10,
removeOnFail: 30,
});
this.systemQueue.add('bakeBufferedReactions', {
await this.systemQueue.add('bakeBufferedReactions', {
}, {
repeat: { pattern: '0 0 * * *' },
removeOnComplete: 10,
removeOnFail: 30,
});
this.systemQueue.add('checkModeratorsActivity', {
await this.systemQueue.add('checkModeratorsActivity', {
}, {
// 毎時30分に起動
repeat: { pattern: '30 * * * *' },
@ -788,7 +794,7 @@ export class QueueService {
userId: webhook.userId,
to: webhook.url,
secret: webhook.secret,
createdAt: Date.now(),
createdAt: this.timeService.now,
eventId: randomUUID(),
};
@ -825,7 +831,7 @@ export class QueueService {
webhookId: webhook.id,
to: webhook.url,
secret: webhook.secret,
createdAt: Date.now(),
createdAt: this.timeService.now,
eventId: randomUUID(),
};
@ -890,7 +896,7 @@ export class QueueService {
@bindThis
public async queueRetryJob(queueType: typeof QUEUE_TYPES[number], jobId: string) {
const queue = this.getQueue(queueType);
const job: Bull.Job | null = await queue.getJob(jobId);
const job: Bull.Job | undefined = await queue.getJob(jobId);
if (job) {
if (job.finishedOn != null) {
await job.retry();
@ -903,7 +909,7 @@ export class QueueService {
@bindThis
public async queueRemoveJob(queueType: typeof QUEUE_TYPES[number], jobId: string) {
const queue = this.getQueue(queueType);
const job: Bull.Job | null = await queue.getJob(jobId);
const job: Bull.Job | undefined = await queue.getJob(jobId);
if (job) {
await job.remove();
}
@ -937,7 +943,7 @@ export class QueueService {
@bindThis
public async queueGetJob(queueType: typeof QUEUE_TYPES[number], jobId: string) {
const queue = this.getQueue(queueType);
const job: Bull.Job | null = await queue.getJob(jobId);
const job: Bull.Job | undefined = await queue.getJob(jobId);
if (job) {
return this.packJobData(job);
} else {

211
packages/backend/src/core/ReactionService.ts
View file

@ -3,11 +3,13 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { DI } from '@/di-symbols.js';
import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository, NoteThreadMutingsRepository, MiMeta } from '@/models/_.js';
import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository, MiMeta } from '@/models/_.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import type { MiRemoteUser, MiUser } from '@/models/User.js';
import { isLocalUser, isRemoteUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import { IdService } from '@/core/IdService.js';
import { MiNoteReaction } from '@/models/NoteReaction.js';
@ -17,13 +19,11 @@ import { NotificationService } from '@/core/NotificationService.js';
import PerUserReactionsChart from '@/core/chart/charts/per-user-reactions.js';
import { emojiRegex } from '@/misc/emoji-regex.js';
import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { bindThis } from '@/decorators.js';
import { UtilityService } from '@/core/UtilityService.js';
import { UserBlockingService } from '@/core/UserBlockingService.js';
import { CustomEmojiService } from '@/core/CustomEmojiService.js';
import { CustomEmojiService, encodeEmojiKey } from '@/core/CustomEmojiService.js';
import { RoleService } from '@/core/RoleService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { trackPromise } from '@/misc/promise-tracker.js';
@ -32,6 +32,7 @@ import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js';
import { CacheService } from '@/core/CacheService.js';
import { NoteVisibilityService } from '@/core/NoteVisibilityService.js';
import { TimeService } from '@/global/TimeService.js';
import type { DataSource } from 'typeorm';
const FALLBACK = '\u2764';
@ -71,8 +72,12 @@ const isCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@\.)?:$/u;
const decodeCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@([\w.-]+))?:$/u;
@Injectable()
export class ReactionService {
export class ReactionService implements OnModuleInit {
private roleService: RoleService;
constructor(
private readonly moduleRef: ModuleRef,
@Inject(DI.meta)
private meta: MiMeta,
@ -85,9 +90,6 @@ export class ReactionService {
@Inject(DI.noteReactionsRepository)
private noteReactionsRepository: NoteReactionsRepository,
@Inject(DI.noteThreadMutingsRepository)
private noteThreadMutingsRepository: NoteThreadMutingsRepository,
@Inject(DI.emojisRepository)
private emojisRepository: EmojisRepository,
@ -96,9 +98,6 @@ export class ReactionService {
private utilityService: UtilityService,
private customEmojiService: CustomEmojiService,
private roleService: RoleService,
private userEntityService: UserEntityService,
private noteEntityService: NoteEntityService,
private userBlockingService: UserBlockingService,
private reactionsBufferingService: ReactionsBufferingService,
private idService: IdService,
@ -110,9 +109,15 @@ export class ReactionService {
private perUserReactionsChart: PerUserReactionsChart,
private readonly cacheService: CacheService,
private readonly noteVisibilityService: NoteVisibilityService,
private readonly timeService: TimeService,
) {
}
@bindThis
onModuleInit() {
this.roleService = this.moduleRef.get('RoleService');
}
@bindThis
public async create(user: { id: MiUser['id']; host: MiUser['host']; isBot: MiUser['isBot'] }, note: MiNote, _reaction?: string | null) {
// Check blocking
@ -144,12 +149,8 @@ export class ReactionService {
const reacterHost = this.utilityService.toPunyNullable(user.host);
const name = custom[1];
const emoji = reacterHost == null
? (await this.customEmojiService.localEmojisCache.fetch()).get(name)
: await this.emojisRepository.findOneBy({
host: reacterHost,
name,
});
const emojiKey = encodeEmojiKey({ name, host: reacterHost });
const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(emojiKey);
if (emoji) {
if (emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.length === 0 || (await this.roleService.getUserRoles(user.id)).some(r => emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.includes(r.id))) {
@ -223,13 +224,13 @@ export class ReactionService {
.execute();
}
this.usersRepository.update({ id: user.id }, { updatedAt: new Date() });
this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date });
// 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新
if (
Math.random() < 0.3 &&
note.userId !== user.id &&
(Date.now() - this.idService.parse(note.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3
(this.timeService.now - this.idService.parse(note.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3
) {
const author = await this.cacheService.findUserById(note.userId);
if (author.isExplorable) {
@ -256,15 +257,9 @@ export class ReactionService {
// カスタム絵文字リアクションだったら絵文字情報も送る
const decodedReaction = this.decodeReaction(reaction);
const customEmoji = decodedReaction.name == null ? null : decodedReaction.host == null
? (await this.customEmojiService.localEmojisCache.fetch()).get(decodedReaction.name)
: await this.emojisRepository.findOne(
{
where: {
name: decodedReaction.name,
host: decodedReaction.host,
},
});
const customEmojiKey = decodedReaction.name == null ? null : encodeEmojiKey({ name: decodedReaction.name, host: decodedReaction.host ?? null });
const customEmoji = customEmojiKey == null ? null :
await this.customEmojiService.emojisByKeyCache.fetchMaybe(customEmojiKey);
this.globalEventService.publishNoteStream(note.id, 'reacted', {
reaction: decodedReaction.reaction,
@ -290,7 +285,7 @@ export class ReactionService {
}
//#region 配信
if (this.userEntityService.isLocalUser(user) && !note.localOnly) {
if (isLocalUser(user) && !note.localOnly) {
const content = this.apRendererService.addContext(await this.apRendererService.renderLike(record, note));
const dm = this.apDeliverManagerService.createDeliverManager(user, content);
if (note.userHost !== null) {
@ -302,7 +297,7 @@ export class ReactionService {
dm.addFollowersRecipe();
} else if (note.visibility === 'specified') {
const visibleUsers = await Promise.all(note.visibleUserIds.map(id => this.usersRepository.findOneBy({ id })));
for (const u of visibleUsers.filter(u => u && this.userEntityService.isRemoteUser(u))) {
for (const u of visibleUsers.filter(u => u && isRemoteUser(u))) {
dm.addDirectRecipe(u as MiRemoteUser);
}
}
@ -345,7 +340,7 @@ export class ReactionService {
.execute();
}
this.usersRepository.update({ id: user.id }, { updatedAt: new Date() });
this.usersRepository.update({ id: user.id }, { updatedAt: this.timeService.date });
this.globalEventService.publishNoteStream(note.id, 'unreacted', {
reaction: this.decodeReaction(exist.reaction).reaction,
@ -353,7 +348,7 @@ export class ReactionService {
});
//#region 配信
if (this.userEntityService.isLocalUser(user) && !note.localOnly) {
if (isLocalUser(user) && !note.localOnly) {
const content = this.apRendererService.addContext(this.apRendererService.renderUndo(await this.apRendererService.renderLike(exist, note), user));
const dm = this.apDeliverManagerService.createDeliverManager(user, content);
if (note.userHost !== null) {
@ -367,86 +362,102 @@ export class ReactionService {
}
/**
* - 文字列タイプのレガシーな形式のリアクションを現在の形式に変換する
* - ローカルのリアクションのホストを `@.` にする（`decodeReaction()`の効果）
* @deprecated Use the exported function instead
*/
@bindThis
public convertLegacyReaction(reaction: string): string {
reaction = this.decodeReaction(reaction).reaction;
if (Object.keys(legacies).includes(reaction)) return legacies[reaction];
return reaction;
}
readonly convertLegacyReaction = convertLegacyReaction;
// TODO: 廃止
/**
* - 文字列タイプのレガシーな形式のリアクションを現在の形式に変換する
* - ローカルのリアクションのホストを `@.` にする（`decodeReaction()`の効果）
* - データベース上には存在する「0個のリアクションがついている」という情報を削除する
* @deprecated Use the exported function instead
*/
@bindThis
public convertLegacyReactions(reactions: MiNote['reactions']): MiNote['reactions'] {
return Object.entries(reactions)
.filter(([, count]) => {
// `ReactionService.prototype.delete`ではリアクション削除時に、
// `MiNote['reactions']`のエントリの値をデクリメントしているが、
// デクリメントしているだけなのでエントリ自体は0を値として持つ形で残り続ける。
// そのため、この処理がなければ、「0個のリアクションがついている」ということになってしまう。
return count > 0;
})
.map(([reaction, count]) => {
const key = this.convertLegacyReaction(reaction);
readonly convertLegacyReactions = convertLegacyReactions;
return [key, count] as const;
})
.reduce<MiNote['reactions']>((acc, [key, count]) => {
// unchecked indexed access
const prevCount = acc[key] as number | undefined;
/**
* @deprecated Use the exported function instead
*/
readonly normalize = normalize;
acc[key] = (prevCount ?? 0) + count;
/**
* @deprecated Use the exported function instead
*/
readonly decodeReaction = decodeReaction;
}
return acc;
}, {});
/**
* - 文字列タイプのレガシーな形式のリアクションを現在の形式に変換する
* - ローカルのリアクションのホストを `@.` にする（`decodeReaction()`の効果）
*/
export function convertLegacyReaction(reaction: string): string {
reaction = decodeReaction(reaction).reaction;
if (Object.keys(legacies).includes(reaction)) return legacies[reaction];
return reaction;
}
// TODO: 廃止
/**
* - 文字列タイプのレガシーな形式のリアクションを現在の形式に変換する
* - ローカルのリアクションのホストを `@.` にする（`decodeReaction()`の効果）
* - データベース上には存在する「0個のリアクションがついている」という情報を削除する
*/
export function convertLegacyReactions(reactions: MiNote['reactions']): MiNote['reactions'] {
return Object.entries(reactions)
.filter(([, count]) => {
// `ReactionService.prototype.delete`ではリアクション削除時に、
// `MiNote['reactions']`のエントリの値をデクリメントしているが、
// デクリメントしているだけなのでエントリ自体は0を値として持つ形で残り続ける。
// そのため、この処理がなければ、「0個のリアクションがついている」ということになってしまう。
return count > 0;
})
.map(([reaction, count]) => {
const key = convertLegacyReaction(reaction);
return [key, count] as const;
})
.reduce<MiNote['reactions']>((acc, [key, count]) => {
// unchecked indexed access
const prevCount = acc[key] as number | undefined;
acc[key] = (prevCount ?? 0) + count;
return acc;
}, {});
}
export function normalize(reaction: string | null): string {
if (reaction == null) return FALLBACK;
// 文字列タイプのリアクションを絵文字に変換
if (Object.keys(legacies).includes(reaction)) return legacies[reaction];
// Unicode絵文字
const match = emojiRegex.exec(reaction);
if (match) {
// 合字を含む1つの絵文字
const unicode = match[0];
// 異体字セレクタ除去
return unicode.match('\u200d') ? unicode : unicode.replace(/\ufe0f/g, '');
}
@bindThis
public normalize(reaction: string | null): string {
if (reaction == null) return FALLBACK;
return FALLBACK;
}
// 文字列タイプのリアクションを絵文字に変換
if (Object.keys(legacies).includes(reaction)) return legacies[reaction];
export function decodeReaction(str: string): DecodedReaction {
const custom = str.match(decodeCustomEmojiRegexp);
// Unicode絵文字
const match = emojiRegex.exec(reaction);
if (match) {
// 合字を含む1つの絵文字
const unicode = match[0];
// 異体字セレクタ除去
return unicode.match('\u200d') ? unicode : unicode.replace(/\ufe0f/g, '');
}
return FALLBACK;
}
@bindThis
public decodeReaction(str: string): DecodedReaction {
const custom = str.match(decodeCustomEmojiRegexp);
if (custom) {
const name = custom[1];
const host = custom[2] ?? null;
return {
reaction: `:${name}@${host ?? '.'}:`, // ローカル分は@以降を省略するのではなく.にする
name,
host,
};
}
if (custom) {
const name = custom[1];
const host = custom[2] ?? null;
return {
reaction: str,
name: undefined,
host: undefined,
reaction: `:${name}@${host ?? '.'}:`, // ローカル分は@以降を省略するのではなく.にする
name,
host,
};
}
return {
reaction: str,
name: undefined,
host: undefined,
};
}

7
packages/backend/src/core/ReactionsBufferingService.ts
View file

@ -6,6 +6,7 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { TimeService } from '@/global/TimeService.js';
import type { MiNote } from '@/models/Note.js';
import { bindThis } from '@/decorators.js';
import type { MiUser, NotesRepository } from '@/models/_.js';
@ -31,6 +32,8 @@ export class ReactionsBufferingService implements OnApplicationShutdown {
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private readonly timeService: TimeService,
) {
this.redisForSub.on('message', this.onMessage);
}
@ -62,7 +65,7 @@ export class ReactionsBufferingService implements OnApplicationShutdown {
for (let i = 0; i < currentPairs.length; i++) {
pipeline.zadd(`${REDIS_PAIR_PREFIX}:${noteId}`, i, currentPairs[i]);
}
pipeline.zadd(`${REDIS_PAIR_PREFIX}:${noteId}`, Date.now(), `${userId}/${reaction}`);
pipeline.zadd(`${REDIS_PAIR_PREFIX}:${noteId}`, this.timeService.now, `${userId}/${reaction}`);
pipeline.zremrangebyrank(`${REDIS_PAIR_PREFIX}:${noteId}`, 0, -(PER_NOTE_REACTION_USER_PAIR_CACHE_MAX + 1));
await pipeline.exec();
}
@ -144,7 +147,7 @@ export class ReactionsBufferingService implements OnApplicationShutdown {
// TODO: scanは重い可能性があるので、別途 bufferedNoteIds を直接Redis上に持っておいてもいいかもしれない
@bindThis
public async bake(): Promise<void> {
const bufferedNoteIds = [];
const bufferedNoteIds: string[] = [];
let cursor = '0';
do {
// https://github.com/redis/ioredis#transparent-key-prefixing

4
packages/backend/src/core/RegistryApiService.ts
View file

@ -11,6 +11,7 @@ import type { MiUser } from '@/models/User.js';
import { IdService } from '@/core/IdService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import { TimeService } from '@/global/TimeService.js';
@Injectable()
export class RegistryApiService {
@ -20,6 +21,7 @@ export class RegistryApiService {
private idService: IdService,
private globalEventService: GlobalEventService,
private readonly timeService: TimeService,
) {
}
@ -31,7 +33,7 @@ export class RegistryApiService {
.insert()
.values({
id: this.idService.gen(),
updatedAt: new Date(),
updatedAt: this.timeService.date,
userId: userId,
domain: domain,
scope: scope,

7
packages/backend/src/core/RelayService.ts
View file

@ -15,10 +15,11 @@ import { DI } from '@/di-symbols.js';
import { deepClone } from '@/misc/clone.js';
import { bindThis } from '@/decorators.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { CacheManagementService, ManagedMemorySingleCache } from '@/global/CacheManagementService.js';
@Injectable()
export class RelayService {
private relaysCache: MemorySingleCache<MiRelay[]>;
private readonly relaysCache: ManagedMemorySingleCache<MiRelay[]>;
constructor(
@Inject(DI.relaysRepository)
@ -28,8 +29,10 @@ export class RelayService {
private queueService: QueueService,
private systemAccountService: SystemAccountService,
private apRendererService: ApRendererService,
cacheManagementService: CacheManagementService,
) {
this.relaysCache = new MemorySingleCache<MiRelay[]>(1000 * 60 * 10); // 10m
this.relaysCache = cacheManagementService.createMemorySingleCache<MiRelay[]>('relay', 1000 * 60 * 10); // 10m
}
@bindThis

7
packages/backend/src/core/RemoteUserResolveService.ts
View file

@ -16,6 +16,7 @@ import { ILink, WebfingerService } from '@/core/WebfingerService.js';
import { RemoteLoggerService } from '@/core/RemoteLoggerService.js';
import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js';
import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
import { TimeService } from '@/global/TimeService.js';
import { bindThis } from '@/decorators.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
@ -35,6 +36,7 @@ export class RemoteUserResolveService {
private remoteLoggerService: RemoteLoggerService,
private apDbResolverService: ApDbResolverService,
private apPersonService: ApPersonService,
private readonly timeService: TimeService,
) {
this.logger = this.remoteLoggerService.logger.createSubLogger('resolve-user');
}
@ -81,10 +83,10 @@ export class RemoteUserResolveService {
}
// ユーザー情報が古い場合は、WebFingerからやりなおして返す
if (user.lastFetchedAt == null || Date.now() - user.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) {
if (user.lastFetchedAt == null || this.timeService.now - user.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) {
// 繋がらないインスタンスに何回も試行するのを防ぐ, 後続の同様処理の連続試行を防ぐ ため 試行前にも更新する
await this.usersRepository.update(user.id, {
lastFetchedAt: new Date(),
lastFetchedAt: this.timeService.date,
});
const self = await this.resolveSelf(acctLower);
@ -105,6 +107,7 @@ export class RemoteUserResolveService {
}, {
uri: self.href,
});
await this.apPersonService.uriPersonCache.delete(user.uri); // Unmap the old URI
}
this.logger.info(`Corrected URI for ${acctLower} from ${user.uri} to ${self.href}`);

39
packages/backend/src/core/ReversiService.ts
View file

@ -16,11 +16,11 @@ import type {
import type { MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { CacheService } from '@/core/CacheService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { IdService } from '@/core/IdService.js';
import { NotificationService } from '@/core/NotificationService.js';
import { TimeService } from '@/global/TimeService.js';
import type { NotificationService } from '@/core/NotificationService.js';
import { Serialized } from '@/types.js';
import { ReversiGameEntityService } from './entities/ReversiGameEntityService.js';
import type { OnApplicationShutdown, OnModuleInit } from '@nestjs/common';
@ -40,16 +40,17 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
@Inject(DI.reversiGamesRepository)
private reversiGamesRepository: ReversiGamesRepository,
private cacheService: CacheService,
private userEntityService: UserEntityService,
private globalEventService: GlobalEventService,
private reversiGameEntityService: ReversiGameEntityService,
private idService: IdService,
private readonly timeService: TimeService,
) {
}
@bindThis
async onModuleInit() {
this.notificationService = this.moduleRef.get(NotificationService.name);
this.notificationService = this.moduleRef.get('NotificationService');
}
@bindThis
@ -100,8 +101,8 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
// 既にマッチしている対局が無いか探す(3分以内)
const games = await this.reversiGamesRepository.find({
where: [
{ id: MoreThan(this.idService.gen(Date.now() - 1000 * 60 * 3)), user1Id: me.id, user2Id: targetUser.id, isStarted: false },
{ id: MoreThan(this.idService.gen(Date.now() - 1000 * 60 * 3)), user1Id: targetUser.id, user2Id: me.id, isStarted: false },
{ id: MoreThan(this.idService.gen(this.timeService.now - 1000 * 60 * 3)), user1Id: me.id, user2Id: targetUser.id, isStarted: false },
{ id: MoreThan(this.idService.gen(this.timeService.now - 1000 * 60 * 3)), user1Id: targetUser.id, user2Id: me.id, isStarted: false },
],
relations: ['user1', 'user2'],
order: { id: 'DESC' },
@ -114,7 +115,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
//#region 相手から既に招待されてないか確認
const invitations = await this.redisClient.zrange(
`reversi:matchSpecific:${me.id}`,
Date.now() - INVITATION_TIMEOUT_MS,
this.timeService.now - INVITATION_TIMEOUT_MS,
'+inf',
'BYSCORE');
@ -130,7 +131,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
//#endregion
const redisPipeline = this.redisClient.pipeline();
redisPipeline.zadd(`reversi:matchSpecific:${targetUser.id}`, Date.now(), me.id);
redisPipeline.zadd(`reversi:matchSpecific:${targetUser.id}`, this.timeService.now, me.id);
redisPipeline.expire(`reversi:matchSpecific:${targetUser.id}`, 120, 'NX');
await redisPipeline.exec();
@ -147,8 +148,8 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
// 既にマッチしている対局が無いか探す(3分以内)
const games = await this.reversiGamesRepository.find({
where: [
{ id: MoreThan(this.idService.gen(Date.now() - 1000 * 60 * 3)), user1Id: me.id, isStarted: false },
{ id: MoreThan(this.idService.gen(Date.now() - 1000 * 60 * 3)), user2Id: me.id, isStarted: false },
{ id: MoreThan(this.idService.gen(this.timeService.now - 1000 * 60 * 3)), user1Id: me.id, isStarted: false },
{ id: MoreThan(this.idService.gen(this.timeService.now - 1000 * 60 * 3)), user2Id: me.id, isStarted: false },
],
relations: ['user1', 'user2'],
order: { id: 'DESC' },
@ -161,7 +162,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
//#region まず自分宛ての招待を探す
const invitations = await this.redisClient.zrange(
`reversi:matchSpecific:${me.id}`,
Date.now() - INVITATION_TIMEOUT_MS,
this.timeService.now - INVITATION_TIMEOUT_MS,
'+inf',
'BYSCORE');
@ -202,9 +203,9 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
} else {
const redisPipeline = this.redisClient.pipeline();
if (options.noIrregularRules) {
redisPipeline.zadd('reversi:matchAny', Date.now(), me.id + ':noIrregularRules');
redisPipeline.zadd('reversi:matchAny', this.timeService.now, me.id + ':noIrregularRules');
} else {
redisPipeline.zadd('reversi:matchAny', Date.now(), me.id);
redisPipeline.zadd('reversi:matchAny', this.timeService.now, me.id);
}
redisPipeline.expire('reversi:matchAny', 15, 'NX');
await redisPipeline.exec();
@ -225,7 +226,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
@bindThis
public async cleanOutdatedGames() {
await this.reversiGamesRepository.delete({
id: LessThan(this.idService.gen(Date.now() - 1000 * 60 * 10)),
id: LessThan(this.idService.gen(this.timeService.now - 1000 * 60 * 10)),
isStarted: false,
});
}
@ -270,7 +271,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
if (isBothReady) {
// 3秒後、両者readyならゲーム開始
setTimeout(async () => {
this.timeService.startTimer(async () => {
const freshGame = await this.get(game.id);
if (freshGame == null || freshGame.isStarted || freshGame.isEnded) return;
if (!freshGame.user1Ready || !freshGame.user2Ready) return;
@ -324,7 +325,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
const updatedGame = await this.reversiGamesRepository.createQueryBuilder().update()
.set({
...this.getBakeProps(game),
startedAt: new Date(),
startedAt: this.timeService.date,
isStarted: true,
black: bw,
map: game.map,
@ -369,7 +370,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
.set({
...this.getBakeProps(game),
isEnded: true,
endedAt: new Date(),
endedAt: this.timeService.date,
winnerId: winnerId,
surrenderedUserId: reason === 'surrender' ? (winnerId === game.user1Id ? game.user2Id : game.user1Id) : null,
timeoutUserId: reason === 'timeout' ? (winnerId === game.user1Id ? game.user2Id : game.user1Id) : null,
@ -393,7 +394,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
public async getInvitations(user: MiUser): Promise<MiUser['id'][]> {
const invitations = await this.redisClient.zrange(
`reversi:matchSpecific:${user.id}`,
Date.now() - INVITATION_TIMEOUT_MS,
this.timeService.now - INVITATION_TIMEOUT_MS,
'+inf',
'BYSCORE');
return invitations;
@ -476,7 +477,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
const logs = Reversi.Serializer.deserializeLogs(game.logs);
const log = {
time: Date.now(),
time: this.timeService.now,
player: myColor,
operation: 'put',
pos,

59
packages/backend/src/core/RoleService.ts
View file

@ -17,19 +17,24 @@ import type {
} from '@/models/_.js';
import { MemoryKVCache, MemorySingleCache } from '@/misc/cache.js';
import type { MiUser } from '@/models/User.js';
import { isLocalUser, isRemoteUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { CacheService } from '@/core/CacheService.js';
import type { FollowStats } from '@/core/CacheService.js';
import type { CacheService, FollowStats } from '@/core/CacheService.js';
import type { RoleCondFormulaValue } from '@/models/Role.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { IdService } from '@/core/IdService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import type { Packed } from '@/misc/json-schema.js';
import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
import { NotificationService } from '@/core/NotificationService.js';
import type { NotificationService } from '@/core/NotificationService.js';
import { TimeService } from '@/global/TimeService.js';
import {
CacheManagementService,
type ManagedMemorySingleCache,
type ManagedMemoryKVCache,
} from '@/global/CacheManagementService.js';
import type { OnApplicationShutdown, OnModuleInit } from '@nestjs/common';
import { getCallerId } from '@/misc/attach-caller-id.js';
@ -115,10 +120,14 @@ export const DEFAULT_POLICIES: RolePolicies = {
canViewFederation: true,
};
// TODO cache sync fixes (and maybe events too?)
@Injectable()
export class RoleService implements OnApplicationShutdown, OnModuleInit {
private rolesCache: MemorySingleCache<MiRole[]>;
private roleAssignmentByUserIdCache: MemoryKVCache<MiRoleAssignment[]>;
private readonly rolesCache: ManagedMemorySingleCache<MiRole[]>;
private readonly roleAssignmentByUserIdCache: ManagedMemoryKVCache<MiRoleAssignment[]>;
private cacheService: CacheService;
private notificationService: NotificationService;
public static AlreadyAssignedError = class extends Error {};
@ -145,22 +154,25 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
@Inject(DI.roleAssignmentsRepository)
private roleAssignmentsRepository: RoleAssignmentsRepository,
private cacheService: CacheService,
private userEntityService: UserEntityService,
private globalEventService: GlobalEventService,
private idService: IdService,
private moderationLogService: ModerationLogService,
private fanoutTimelineService: FanoutTimelineService,
private readonly timeService: TimeService,
cacheManagementService: CacheManagementService,
) {
this.rolesCache = new MemorySingleCache<MiRole[]>(1000 * 60 * 60); // 1h
this.roleAssignmentByUserIdCache = new MemoryKVCache<MiRoleAssignment[]>(1000 * 60 * 5); // 5m
this.rolesCache = cacheManagementService.createMemorySingleCache<MiRole[]>('roles', 1000 * 60 * 60); // 1h
this.roleAssignmentByUserIdCache = cacheManagementService.createMemoryKVCache<MiRoleAssignment[]>('roleAssignment', 1000 * 60 * 5); // 5m
// TODO additional cache for final calculation?
this.redisForSub.on('message', this.onMessage);
}
@bindThis
async onModuleInit() {
this.notificationService = this.moduleRef.get(NotificationService.name);
this.notificationService = this.moduleRef.get('NotificationService');
this.cacheService = this.moduleRef.get('CacheService');
}
@bindThis
@ -249,11 +261,11 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
}
// ローカルユーザのみ
case 'isLocal': {
return this.userEntityService.isLocalUser(user);
return isLocalUser(user);
}
// リモートユーザのみ
case 'isRemote': {
return this.userEntityService.isRemoteUser(user);
return isRemoteUser(user);
}
// User is from a specific instance
case 'isFromInstance': {
@ -294,11 +306,11 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
}
// ユーザが作成されてから指定期間経過した
case 'createdLessThan': {
return this.idService.parse(user.id).date.getTime() > (Date.now() - (value.sec * 1000));
return this.idService.parse(user.id).date.getTime() > (this.timeService.now - (value.sec * 1000));
}
// ユーザが作成されてから指定期間経っていない
case 'createdMoreThan': {
return this.idService.parse(user.id).date.getTime() < (Date.now() - (value.sec * 1000));
return this.idService.parse(user.id).date.getTime() < (this.timeService.now - (value.sec * 1000));
}
// フォロワー数が指定値以下
case 'followersLessThanOrEq': {
@ -398,7 +410,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
@bindThis
public async getUserAssigns(userOrId: MiUser | MiUser['id']) {
const now = Date.now();
const now = this.timeService.now;
const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId;
let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId }));
// 期限切れのロールを除外
@ -437,7 +449,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
*/
@bindThis
public async getUserBadgeRoles(userOrId: MiUser | MiUser['id']) {
const now = Date.now();
const now = this.timeService.now;
const userId = typeof(userOrId) === 'object' ? userOrId.id : userOrId;
let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId }));
// 期限切れのロールを除外
@ -583,7 +595,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
: [];
// Setを経由して重複を除去（ユーザIDは重複する可能性があるので）
const now = Date.now();
const now = this.timeService.now;
const resultSet = new Set(
assigns
.filter(it =>
@ -637,7 +649,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
@bindThis
public async assign(userId: MiUser['id'], roleId: MiRole['id'], expiresAt: Date | null = null, moderator?: MiUser): Promise<void> {
const now = Date.now();
const now = this.timeService.now;
const role = await this.rolesRepository.findOneByOrFail({ id: roleId });
@ -665,7 +677,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
});
this.rolesRepository.update(roleId, {
lastUsedAt: new Date(),
lastUsedAt: this.timeService.date,
});
this.globalEventService.publishInternalEvent('userRoleAssigned', created);
@ -692,7 +704,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
@bindThis
public async unassign(userId: MiUser['id'], roleId: MiRole['id'], moderator?: MiUser): Promise<void> {
const now = new Date();
const now = this.timeService.date;
const existing = await this.roleAssignmentsRepository.findOneBy({ roleId, userId });
if (existing == null) {
@ -744,7 +756,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
@bindThis
public async create(values: Partial<MiRole>, moderator?: MiUser): Promise<MiRole> {
const date = new Date();
const date = this.timeService.date;
const created = await this.rolesRepository.insertOne({
id: this.idService.gen(date.getTime()),
updatedAt: date,
@ -780,7 +792,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
@bindThis
public async update(role: MiRole, params: Partial<MiRole>, moderator?: MiUser): Promise<void> {
const date = new Date();
const date = this.timeService.date;
await this.rolesRepository.update(role.id, {
updatedAt: date,
...params,
@ -825,7 +837,6 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.roleAssignmentByUserIdCache.dispose();
}
@bindThis

70
packages/backend/src/core/S3Service.ts
View file

@ -6,24 +6,61 @@
import { URL } from 'node:url';
import * as http from 'node:http';
import * as https from 'node:https';
import { Injectable } from '@nestjs/common';
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import { DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { Upload } from '@aws-sdk/lib-storage';
import { NodeHttpHandler, NodeHttpHandlerOptions } from '@smithy/node-http-handler';
import type { MiMeta } from '@/models/Meta.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { DI } from '@/di-symbols.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import type { InternalEventTypes } from '@/core/GlobalEventService.js';
import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/client-s3';
@Injectable()
export class S3Service {
export class S3Service implements OnApplicationShutdown {
private client?: S3Client;
constructor(
@Inject(DI.meta)
private readonly meta: MiMeta,
private httpRequestService: HttpRequestService,
private readonly internalEventService: InternalEventService,
) {
this.internalEventService.on('metaUpdated', this.onMetaUpdated);
}
@bindThis
public getS3Client(meta: MiMeta): S3Client {
private onMetaUpdated(body: InternalEventTypes['metaUpdated']): void {
if (this.needsChange(body.before, body.after)) {
this.disposeClient();
this.client = this.createS3Client(body.after);
}
}
private needsChange(before: MiMeta | undefined, after: MiMeta): boolean {
if (before == null) return true;
if (before.objectStorageEndpoint !== after.objectStorageEndpoint) return true;
if (before.objectStorageUseSSL !== after.objectStorageUseSSL) return true;
if (before.objectStorageUseProxy !== after.objectStorageUseProxy) return true;
if (before.objectStorageAccessKey !== after.objectStorageAccessKey) return true;
if (before.objectStorageSecretKey !== after.objectStorageSecretKey) return true;
if (before.objectStorageRegion !== after.objectStorageRegion) return true;
if (before.objectStorageUseSSL !== after.objectStorageUseSSL) return true;
if (before.objectStorageS3ForcePathStyle !== after.objectStorageS3ForcePathStyle) return true;
if (before.objectStorageRegion !== after.objectStorageRegion) return true;
return false;
}
@bindThis
private getS3Client(): S3Client {
return this.client ??= this.createS3Client(this.meta);
}
@bindThis
private createS3Client(meta: MiMeta): S3Client {
const u = meta.objectStorageEndpoint
? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}`
: `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
@ -52,8 +89,8 @@ export class S3Service {
}
@bindThis
public async upload(meta: MiMeta, input: PutObjectCommandInput) {
const client = this.getS3Client(meta);
public async upload(input: PutObjectCommandInput) {
const client = this.getS3Client();
return new Upload({
client,
params: input,
@ -64,8 +101,27 @@ export class S3Service {
}
@bindThis
public delete(meta: MiMeta, input: DeleteObjectCommandInput) {
const client = this.getS3Client(meta);
public delete(input: DeleteObjectCommandInput) {
const client = this.getS3Client();
return client.send(new DeleteObjectCommand(input));
}
@bindThis
private disposeClient(): void {
if (this.client) {
this.client.destroy();
this.client = undefined;
}
}
@bindThis
private dispose(): void {
this.disposeClient();
this.internalEventService.off('metaUpdated', this.onMetaUpdated);
}
@bindThis
onApplicationShutdown() {
this.dispose();
}
}

4
packages/backend/src/core/SignupService.ts
View file

@ -23,6 +23,7 @@ import { UtilityService } from '@/core/UtilityService.js';
import { UserService } from '@/core/UserService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { MetaService } from '@/core/MetaService.js';
import { TimeService } from '@/global/TimeService.js';
@Injectable()
export class SignupService {
@ -46,6 +47,7 @@ export class SignupService {
private systemAccountService: SystemAccountService,
private metaService: MetaService,
private usersChart: UsersChart,
private readonly timeService: TimeService,
) {
}
@ -150,7 +152,7 @@ export class SignupService {
}));
await transactionalEntityManager.save(new MiUsedUsername({
createdAt: new Date(),
createdAt: this.timeService.date,
username: username.toLowerCase(),
}));
});

19
packages/backend/src/core/SponsorsService.ts
View file

@ -3,12 +3,11 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import { Inject, Injectable } from '@nestjs/common';
import type { MiMeta } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { RedisKVCache } from '@/misc/cache.js';
import { bindThis } from '@/decorators.js';
import { CacheManagementService, type ManagedRedisKVCache } from '@/global/CacheManagementService.js';
export interface Sponsor {
MemberId: number;
@ -34,17 +33,16 @@ export interface Sponsor {
}
@Injectable()
export class SponsorsService implements OnApplicationShutdown {
private readonly cache: RedisKVCache<Sponsor[]>;
export class SponsorsService {
private readonly cache: ManagedRedisKVCache<Sponsor[]>;
constructor(
@Inject(DI.meta)
private readonly meta: MiMeta,
@Inject(DI.redis)
redisClient: Redis.Redis,
cacheManagementService: CacheManagementService,
) {
this.cache = new RedisKVCache<Sponsor[]>(redisClient, 'sponsors', {
this.cache = cacheManagementService.createRedisKVCache<Sponsor[]>('sponsors', {
lifetime: 1000 * 60 * 60,
memoryCacheLifetime: 1000 * 60,
fetcher: (key) => {
@ -102,9 +100,4 @@ export class SponsorsService implements OnApplicationShutdown {
if (forceUpdate) await this.cache.refresh('sharkey');
return this.cache.fetch('sharkey');
}
@bindThis
public onApplicationShutdown(): void {
this.cache.dispose();
}
}

56
packages/backend/src/core/SystemAccountService.ts
View file

@ -19,12 +19,16 @@ import { bindThis } from '@/decorators.js';
import { generateNativeUserToken } from '@/misc/token.js';
import { IdService } from '@/core/IdService.js';
import { genRsaKeyPair } from '@/misc/gen-key-pair.js';
import { CacheManagementService, type ManagedMemoryKVCache } from '@/global/CacheManagementService.js';
import { CacheService } from '@/core/CacheService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import { TimeService } from '@/global/TimeService.js';
export const SYSTEM_ACCOUNT_TYPES = ['actor', 'relay', 'proxy'] as const;
@Injectable()
export class SystemAccountService implements OnApplicationShutdown {
private cache: MemoryKVCache<MiLocalUser>;
private readonly cache: ManagedMemoryKVCache<string>;
constructor(
@Inject(DI.redisForSub)
@ -46,8 +50,13 @@ export class SystemAccountService implements OnApplicationShutdown {
private userProfilesRepository: UserProfilesRepository,
private idService: IdService,
private readonly cacheService: CacheService,
private readonly internalEventService: InternalEventService,
private readonly timeService: TimeService,
cacheManagementService: CacheManagementService,
) {
this.cache = new MemoryKVCache<MiLocalUser>(1000 * 60 * 10); // 10m
this.cache = cacheManagementService.createMemoryKVCache<string>('systemAccount', 1000 * 60 * 10); // 10m
this.redisForSub.on('message', this.onMessage);
}
@ -84,25 +93,26 @@ export class SystemAccountService implements OnApplicationShutdown {
@bindThis
public async fetch(type: typeof SYSTEM_ACCOUNT_TYPES[number]): Promise<MiLocalUser> {
const cached = this.cache.get(type);
if (cached) return cached;
// Use local cache to find userId for type
const userId = await this.cache.fetch(type, async () => {
const systemAccount = await this.systemAccountsRepository.findOne({
where: { type: type },
select: { userId: true },
}) as { userId: string } | null;
const systemAccount = await this.systemAccountsRepository.findOne({
where: { type: type },
relations: ['user'],
if (systemAccount) {
return systemAccount.userId;
} else {
const created = await this.createCorrespondingUser(type, {
username: `system.${type}`, // NOTE: (できれば避けたいが) . が含まれるかどうかでシステムアカウントかどうかを判定している処理もあるので変えないように
name: this.meta.name,
});
return created.id;
}
});
if (systemAccount) {
this.cache.set(type, systemAccount.user as MiLocalUser);
return systemAccount.user as MiLocalUser;
} else {
const created = await this.createCorrespondingUser(type, {
username: `system.${type}`, // NOTE: (できれば避けたいが) . が含まれるかどうかでシステムアカウントかどうかを判定している処理もあるので変えないように
name: this.meta.name,
});
this.cache.set(type, created);
return created;
}
// Get the actual user entity from shared caches.
return await this.cacheService.findLocalUserById(userId);
}
@bindThis
@ -166,7 +176,7 @@ export class SystemAccountService implements OnApplicationShutdown {
});
await transactionalEntityManager.insert(MiUsedUsername, {
createdAt: new Date(),
createdAt: this.timeService.date,
username: extra.username.toLowerCase(),
});
@ -192,6 +202,7 @@ export class SystemAccountService implements OnApplicationShutdown {
if (Object.keys(updates).length > 0) {
await this.usersRepository.update(user.id, updates);
await this.internalEventService.emit('localUserUpdated', { id: user.id });
}
const profileUpdates = {} as Partial<MiUserProfile>;
@ -199,12 +210,10 @@ export class SystemAccountService implements OnApplicationShutdown {
if (Object.keys(profileUpdates).length > 0) {
await this.userProfilesRepository.update(user.id, profileUpdates);
await this.internalEventService.emit('updateUserProfile', { userId: user.id });
}
const updated = await this.usersRepository.findOneByOrFail({ id: user.id }) as MiLocalUser;
this.cache.set(type, updated);
return updated;
return await this.cacheService.findLocalUserById(user.id);
}
public async getInstanceActor() {
@ -222,7 +231,6 @@ export class SystemAccountService implements OnApplicationShutdown {
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.cache.dispose();
}
@bindThis

73
packages/backend/src/core/SystemWebhookService.ts
View file

@ -18,6 +18,9 @@ import Logger from '@/logger.js';
import { Packed } from '@/misc/json-schema.js';
import { AbuseReportResolveType } from '@/models/AbuseUserReport.js';
import { ModeratorInactivityRemainingTime } from '@/queue/processors/CheckModeratorsActivityProcessorService.js';
import { CacheManagementService, type ManagedMemorySingleCache } from '@/global/CacheManagementService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import { TimeService } from '@/global/TimeService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
export type AbuseReportPayload = {
@ -50,8 +53,7 @@ export type SystemWebhookPayload<T extends SystemWebhookEventType> =
@Injectable()
export class SystemWebhookService implements OnApplicationShutdown {
private activeSystemWebhooksFetched = false;
private activeSystemWebhooks: MiSystemWebhook[] = [];
private readonly activeSystemWebhooks: ManagedMemorySingleCache<MiSystemWebhook[]>;
constructor(
@Inject(DI.redisForSub)
@ -62,20 +64,25 @@ export class SystemWebhookService implements OnApplicationShutdown {
private queueService: QueueService,
private moderationLogService: ModerationLogService,
private globalEventService: GlobalEventService,
private readonly internalEventService: InternalEventService,
private readonly timeService: TimeService,
cacheManagementService: CacheManagementService,
) {
this.redisForSub.on('message', this.onMessage);
this.activeSystemWebhooks = cacheManagementService.createMemorySingleCache<MiSystemWebhook[]>('systemWebhooks', 1000 * 60 * 60 * 12); // 12h
this.internalEventService.on('systemWebhookCreated', this.onWebhookEvent);
this.internalEventService.on('systemWebhookUpdated', this.onWebhookEvent);
this.internalEventService.on('systemWebhookDeleted', this.onWebhookEvent);
}
@bindThis
public async fetchActiveSystemWebhooks() {
if (!this.activeSystemWebhooksFetched) {
this.activeSystemWebhooks = await this.systemWebhooksRepository.findBy({
return await this.activeSystemWebhooks.fetch(async () => {
return await this.systemWebhooksRepository.findBy({
isActive: true,
});
this.activeSystemWebhooksFetched = true;
}
return this.activeSystemWebhooks;
});
}
/**
@ -124,7 +131,7 @@ export class SystemWebhookService implements OnApplicationShutdown {
});
const webhook = await this.systemWebhooksRepository.findOneByOrFail({ id });
this.globalEventService.publishInternalEvent('systemWebhookCreated', webhook);
await this.internalEventService.emit('systemWebhookCreated', { id: webhook.id });
this.moderationLogService
.log(updater, 'createSystemWebhook', {
systemWebhookId: webhook.id,
@ -151,7 +158,7 @@ export class SystemWebhookService implements OnApplicationShutdown {
): Promise<MiSystemWebhook> {
const beforeEntity = await this.systemWebhooksRepository.findOneByOrFail({ id: params.id });
await this.systemWebhooksRepository.update(beforeEntity.id, {
updatedAt: new Date(),
updatedAt: this.timeService.date,
isActive: params.isActive,
name: params.name,
on: params.on,
@ -160,7 +167,7 @@ export class SystemWebhookService implements OnApplicationShutdown {
});
const afterEntity = await this.systemWebhooksRepository.findOneByOrFail({ id: beforeEntity.id });
this.globalEventService.publishInternalEvent('systemWebhookUpdated', afterEntity);
await this.internalEventService.emit('systemWebhookUpdated', { id: afterEntity.id });
this.moderationLogService
.log(updater, 'updateSystemWebhook', {
systemWebhookId: beforeEntity.id,
@ -179,7 +186,7 @@ export class SystemWebhookService implements OnApplicationShutdown {
const webhook = await this.systemWebhooksRepository.findOneByOrFail({ id });
await this.systemWebhooksRepository.delete(id);
this.globalEventService.publishInternalEvent('systemWebhookDeleted', webhook);
await this.internalEventService.emit('systemWebhookDeleted', { id: webhook.id });
this.moderationLogService
.log(updater, 'deleteSystemWebhook', {
systemWebhookId: webhook.id,
@ -211,45 +218,15 @@ export class SystemWebhookService implements OnApplicationShutdown {
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel !== 'internal') {
return;
}
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
switch (type) {
case 'systemWebhookCreated': {
if (body.isActive) {
this.activeSystemWebhooks.push(MiSystemWebhook.deserialize(body));
}
break;
}
case 'systemWebhookUpdated': {
if (body.isActive) {
const i = this.activeSystemWebhooks.findIndex(a => a.id === body.id);
if (i > -1) {
this.activeSystemWebhooks[i] = MiSystemWebhook.deserialize(body);
} else {
this.activeSystemWebhooks.push(MiSystemWebhook.deserialize(body));
}
} else {
this.activeSystemWebhooks = this.activeSystemWebhooks.filter(a => a.id !== body.id);
}
break;
}
case 'systemWebhookDeleted': {
this.activeSystemWebhooks = this.activeSystemWebhooks.filter(a => a.id !== body.id);
break;
}
default:
break;
}
private onWebhookEvent(): void {
this.activeSystemWebhooks.delete();
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.internalEventService.off('systemWebhookCreated', this.onWebhookEvent);
this.internalEventService.off('systemWebhookUpdated', this.onWebhookEvent);
this.internalEventService.off('systemWebhookDeleted', this.onWebhookEvent);
}
@bindThis

27
packages/backend/src/core/TimeService.ts
View file

@ -1,27 +0,0 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Injectable } from '@nestjs/common';
/**
* Provides abstractions to access the current time.
* Exists for unit testing purposes, so that tests can "simulate" any given time for consistency.
*/
@Injectable()
export class TimeService {
/**
* Returns Date.now()
*/
public get now() {
return Date.now();
}
/**
* Returns a new Date instance.
*/
public get date() {
return new Date();
}
}

6
packages/backend/src/core/UpdateInstanceQueue.ts
View file

@ -8,6 +8,7 @@ import { CollapsedQueue } from '@/misc/collapsed-queue.js';
import { bindThis } from '@/decorators.js';
import { MiNote } from '@/models/Note.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { TimeService } from '@/global/TimeService.js';
type UpdateInstanceJob = {
latestRequestReceivedAt: Date,
@ -19,8 +20,9 @@ type UpdateInstanceJob = {
export class UpdateInstanceQueue extends CollapsedQueue<MiNote['id'], UpdateInstanceJob> implements OnApplicationShutdown {
constructor(
private readonly federatedInstanceService: FederatedInstanceService,
timeService: TimeService,
) {
super(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, (id, job) => this.collapseUpdateInstanceJobs(id, job), (id, job) => this.performUpdateInstance(id, job));
super(timeService, process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, (id, job) => this.collapseUpdateInstanceJobs(id, job), (id, job) => this.performUpdateInstance(id, job));
}
@bindThis
@ -38,7 +40,7 @@ export class UpdateInstanceQueue extends CollapsedQueue<MiNote['id'], UpdateInst
@bindThis
private async performUpdateInstance(id: string, job: UpdateInstanceJob) {
await this.federatedInstanceService.update(id, {
latestRequestReceivedAt: new Date(),
latestRequestReceivedAt: this.timeService.date,
isNotResponding: false,
// もしサーバーが死んでるために配信が止まっていた場合には自動的に復活させてあげる
suspensionState: job.shouldUnsuspend ? 'none' : undefined,

17
packages/backend/src/core/UserBlockingService.ts
View file

@ -19,7 +19,8 @@ import { LoggerService } from '@/core/LoggerService.js';
import { UserWebhookService } from '@/core/UserWebhookService.js';
import { bindThis } from '@/decorators.js';
import { CacheService } from '@/core/CacheService.js';
import { UserFollowingService } from '@/core/UserFollowingService.js';
import type { UserFollowingService } from '@/core/UserFollowingService.js';
import { UserListService } from '@/core/UserListService.js';
@Injectable()
export class UserBlockingService implements OnModuleInit {
@ -49,10 +50,12 @@ export class UserBlockingService implements OnModuleInit {
private webhookService: UserWebhookService,
private apRendererService: ApRendererService,
private loggerService: LoggerService,
private readonly userListService: UserListService,
) {
this.logger = this.loggerService.getLogger('user-block');
}
@bindThis
onModuleInit() {
this.userFollowingService = this.moduleRef.get('UserFollowingService');
}
@ -139,16 +142,12 @@ export class UserBlockingService implements OnModuleInit {
@bindThis
private async removeFromList(listOwner: MiUser, user: MiUser) {
const userLists = await this.userListsRepository.findBy({
userId: listOwner.id,
const userLists = await this.userListsRepository.find({
where: { userId: listOwner.id },
select: { id: true },
});
for (const userList of userLists) {
await this.userListMembershipsRepository.delete({
userListId: userList.id,
userId: user.id,
});
}
await this.userListService.bulkRemoveMember(user, userLists.map(l => l.id));
}
@bindThis

5
packages/backend/src/core/UserFollowingService.ts
View file

@ -22,14 +22,14 @@ import type { FollowingsRepository, FollowRequestsRepository, InstancesRepositor
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { bindThis } from '@/decorators.js';
import { UserBlockingService } from '@/core/UserBlockingService.js';
import type { UserBlockingService } from '@/core/UserBlockingService.js';
import { CacheService } from '@/core/CacheService.js';
import type { Config } from '@/config.js';
import { AccountMoveService } from '@/core/AccountMoveService.js';
import { UtilityService } from '@/core/UtilityService.js';
import type { ThinUser } from '@/queue/types.js';
import { LoggerService } from '@/core/LoggerService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import type Logger from '../logger.js';
type Local = MiLocalUser | {
@ -94,6 +94,7 @@ export class UserFollowingService implements OnModuleInit {
this.logger = loggerService.getLogger('following/create');
}
@bindThis
onModuleInit() {
this.userBlockingService = this.moduleRef.get('UserBlockingService');
}

32
packages/backend/src/core/UserKeypairService.ts
View file

@ -5,16 +5,19 @@
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import { In } from 'typeorm';
import type { MiUser } from '@/models/User.js';
import type { UserKeypairsRepository } from '@/models/_.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import type { MiUserKeypair } from '@/models/UserKeypair.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import type { InternalEventTypes } from '@/core/GlobalEventService.js';
@Injectable()
export class UserKeypairService implements OnApplicationShutdown {
private cache: MemoryKVCache<MiUserKeypair>;
public readonly userKeypairCache: ManagedQuantumKVCache<MiUserKeypair>;
constructor(
@Inject(DI.redis)
@ -22,18 +25,37 @@ export class UserKeypairService implements OnApplicationShutdown {
@Inject(DI.userKeypairsRepository)
private userKeypairsRepository: UserKeypairsRepository,
private readonly internalEventService: InternalEventService,
cacheManagementService: CacheManagementService,
) {
this.cache = new MemoryKVCache<MiUserKeypair>(1000 * 60 * 60 * 24); // 24h
this.userKeypairCache = cacheManagementService.createQuantumKVCache('userKeypair', {
lifetime: 1000 * 60 * 60, // 1h
fetcher: async userId => await this.userKeypairsRepository.findOneByOrFail({ userId }),
optionalFetcher: async userId => await this.userKeypairsRepository.findOneBy({ userId }),
bulkFetcher: async userIds => {
const keypairs = await this.userKeypairsRepository.findBy({ userId: In(userIds) });
return keypairs.map(keypair => [keypair.userId, keypair]);
},
});
this.internalEventService.on('userChangeDeletedState', this.onUserDeleted);
}
@bindThis
public async getUserKeypair(userId: MiUser['id']): Promise<MiUserKeypair> {
return await this.cache.fetch(userId, () => this.userKeypairsRepository.findOneByOrFail({ userId }));
return await this.userKeypairCache.fetch(userId);
}
@bindThis
private async onUserDeleted(body: InternalEventTypes['userChangeDeletedState']): Promise<void> {
await this.userKeypairCache.delete(body.id);
}
@bindThis
public dispose(): void {
this.cache.dispose();
this.internalEventService.off('userChangeDeletedState', this.onUserDeleted);
}
@bindThis

178
packages/backend/src/core/UserListService.ts
View file

@ -6,27 +6,29 @@
import { Inject, Injectable, OnApplicationShutdown, OnModuleInit } from '@nestjs/common';
import * as Redis from 'ioredis';
import { ModuleRef } from '@nestjs/core';
import type { MiMeta, UserListMembershipsRepository } from '@/models/_.js';
import { In } from 'typeorm';
import type { MiMeta, UserListMembershipsRepository, UserListsRepository } from '@/models/_.js';
import type { MiUser } from '@/models/User.js';
import type { MiUserList } from '@/models/UserList.js';
import type { MiUserListMembership } from '@/models/UserListMembership.js';
import { IdService } from '@/core/IdService.js';
import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import { QueueService } from '@/core/QueueService.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { RoleService } from '@/core/RoleService.js';
import type { RoleService } from '@/core/RoleService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import { CacheService } from '@/core/CacheService.js';
import { CacheManagementService, type ManagedQuantumKVCache } from '@/global/CacheManagementService.js';
@Injectable()
export class UserListService implements OnApplicationShutdown, OnModuleInit {
export class UserListService implements OnModuleInit {
public static TooManyUsersError = class extends Error {};
public membersCache: QuantumKVCache<Set<string>>;
public readonly userListsCache: ManagedQuantumKVCache<MiUserList>;
private roleService: RoleService;
constructor(
@ -41,6 +43,9 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
@Inject(DI.userListMembershipsRepository)
private userListMembershipsRepository: UserListMembershipsRepository,
@Inject(DI.userListsRepository)
private readonly userListsRepository: UserListsRepository,
@Inject(DI.meta)
private readonly meta: MiMeta,
@ -50,52 +55,33 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
private queueService: QueueService,
private systemAccountService: SystemAccountService,
private readonly internalEventService: InternalEventService,
) {
this.membersCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userListMembers', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: (key) => this.userListMembershipsRepository.find({ where: { userListId: key }, select: ['userId'] }).then(xs => new Set(xs.map(x => x.userId))),
});
private readonly cacheService: CacheService,
this.internalEventService.on('userListMemberAdded', this.onMessage);
this.internalEventService.on('userListMemberRemoved', this.onMessage);
cacheManagementService: CacheManagementService,
) {
this.userListsCache = cacheManagementService.createQuantumKVCache('userLists', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: async id => await this.userListsRepository.findOneByOrFail({ id }),
optionalFetcher: async id => await this.userListsRepository.findOneBy({ id }),
bulkFetcher: async ids => {
const lists = await this.userListsRepository.findBy({ id: In(ids) });
return lists.map(list => [list.id, list]);
},
});
}
@bindThis
async onModuleInit() {
this.roleService = this.moduleRef.get('RoleService');
}
@bindThis
private async onMessage<E extends 'userListMemberAdded' | 'userListMemberRemoved'>(body: InternalEventTypes[E], type: E): Promise<void> {
{
switch (type) {
case 'userListMemberAdded': {
const { userListId, memberId } = body;
const members = this.membersCache.get(userListId);
if (members) {
members.add(memberId);
}
break;
}
case 'userListMemberRemoved': {
const { userListId, memberId } = body;
const members = this.membersCache.get(userListId);
if (members) {
members.delete(memberId);
}
break;
}
default:
break;
}
}
}
@bindThis
public async addMember(target: MiUser, list: MiUserList, me: MiUser) {
const currentCount = await this.userListMembershipsRepository.countBy({
userListId: list.id,
});
if (currentCount >= (await this.roleService.getUserPolicies(me.id)).userEachUserListsLimit) {
const [current, policies] = await Promise.all([
this.cacheService.listUserMembershipsCache.fetch(list.id),
this.roleService.getUserPolicies(me),
]);
if (current.size >= policies.userEachUserListsLimit) {
throw new UserListService.TooManyUsersError();
}
@ -106,13 +92,13 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
userListUserId: list.userId,
} as MiUserListMembership);
this.globalEventService.publishInternalEvent('userListMemberAdded', { userListId: list.id, memberId: target.id });
await this.internalEventService.emit('userListMemberAdded', { userListId: list.id, memberId: target.id });
this.globalEventService.publishUserListStream(list.id, 'userAdded', await this.userEntityService.pack(target));
// このインスタンス内にこのリモートユーザーをフォローしているユーザーがいなくても投稿を受け取るためにダミーのユーザーがフォローしたということにする
if (this.userEntityService.isRemoteUser(target) && this.meta.enableProxyAccount) {
const proxy = await this.systemAccountService.fetch('proxy');
this.queueService.createFollowJob([{ from: { id: proxy.id }, to: { id: target.id } }]);
await this.queueService.createFollowJob([{ from: { id: proxy.id }, to: { id: target.id } }]);
}
}
@ -123,16 +109,15 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
userListId: list.id,
});
this.globalEventService.publishInternalEvent('userListMemberRemoved', { userListId: list.id, memberId: target.id });
await this.internalEventService.emit('userListMemberRemoved', { userListId: list.id, memberId: target.id });
this.globalEventService.publishUserListStream(list.id, 'userRemoved', await this.userEntityService.pack(target));
}
@bindThis
public async updateMembership(target: MiUser, list: MiUserList, options: { withReplies?: boolean }) {
const membership = await this.userListMembershipsRepository.findOneBy({
userId: target.id,
userListId: list.id,
});
const membership = await this.cacheService.userListMembershipsCache
.fetchMaybe(target.id)
.then(ms => ms?.get(list.id));
if (membership == null) {
throw new Error('User is not a member of the list');
@ -143,17 +128,96 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
}, {
withReplies: options.withReplies,
});
await this.internalEventService.emit('userListMemberUpdated', { userListId: list.id, memberId: target.id });
}
@bindThis
public dispose(): void {
this.internalEventService.off('userListMemberAdded', this.onMessage);
this.internalEventService.off('userListMemberRemoved', this.onMessage);
this.membersCache.dispose();
public async bulkAddMember(target: { id: MiUser['id'] }, memberships: { userListId: MiUserList['id'], withReplies?: boolean }[]): Promise<void> {
const userListIds = memberships.map(m => m.userListId);
const userLists = await this.userListsCache.fetchMany(userListIds);
// Map userListId => userListUserId
const listUserIds = new Map(userLists.values.map(l => [l.id, l.userId]));
const toInsert = memberships.map(membership => ({
id: this.idService.gen(),
userId: target.id,
userListId: membership.userListId,
userListUserId: listUserIds.get(membership.userListId),
withReplies: membership.withReplies,
}));
await this.userListMembershipsRepository.insert(toInsert);
await this.internalEventService.emit('userListMemberBulkAdded', {
memberId: target.id,
userListIds,
});
const targetUser = await this.cacheService.findUserById(target.id);
const packedUser = await this.userEntityService.pack(targetUser);
await Promise.all(memberships.map(async membership => {
await this.globalEventService.publishUserListStream(membership.userListId, 'userAdded', packedUser);
}));
// このインスタンス内にこのリモートユーザーをフォローしているユーザーがいなくても投稿を受け取るためにダミーのユーザーがフォローしたということにする
if (this.userEntityService.isRemoteUser(targetUser) && this.meta.enableProxyAccount) {
const proxy = await this.systemAccountService.fetch('proxy');
await this.queueService.createFollowJob([{ from: { id: proxy.id }, to: { id: target.id } }]);
}
}
@bindThis
public onApplicationShutdown(signal?: string | undefined): void {
this.dispose();
public async bulkRemoveMember(target: { id: MiUser['id'] }, memberships: { userListId: MiUserList['id'] }[] | MiUserList['id'][]): Promise<void> {
const userListIds = memberships.map(mem => typeof(mem) === 'object' ? mem.userListId : mem);
await this.userListMembershipsRepository.delete({
userId: target.id,
userListId: In(userListIds),
});
await this.internalEventService.emit('userListMemberBulkRemoved', {
userListIds,
memberId: target.id,
});
const targetUser = await this.cacheService.findUserById(target.id);
const packedUser = await this.userEntityService.pack(targetUser);
await Promise.all(userListIds.map(async userListId => {
await this.globalEventService.publishUserListStream(userListId, 'userRemoved', packedUser);
}));
}
@bindThis
public async bulkUpdateMembership(target: { id: MiUser['id'] }, memberships: { userListId: MiUserList['id'], withReplies: boolean }[]): Promise<void> {
const userListMemberships = await this.cacheService.userListMembershipsCache.fetch(target.id);
const membershipChanges = memberships
.map(mem => {
const old = userListMemberships.get(mem.userListId);
return {
new: mem,
id: old?.id ?? '',
old,
};
});
const toAddReplies = membershipChanges
.filter(mem => mem.old != null && mem.new.withReplies && !mem.old.withReplies)
.map(mem => mem.id);
if (toAddReplies.length > 0) {
await this.userListMembershipsRepository.update({ id: In(toAddReplies) }, { withReplies: true });
}
const toRemoveReplies = membershipChanges
.filter(mem => mem.old != null && !mem.new.withReplies && mem.old.withReplies)
.map(mem => mem.id);
if (toRemoveReplies.length > 0) {
await this.userListMembershipsRepository.update({ id: In(toRemoveReplies) }, { withReplies: false });
}
await this.internalEventService.emit('userListMemberBulkUpdated', {
memberId: target.id,
userListIds: memberships.map(m => m.userListId),
});
}
}

16
packages/backend/src/core/UserSearchService.ts
View file

@ -11,12 +11,9 @@ import { bindThis } from '@/decorators.js';
import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
import type { Config } from '@/config.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { TimeService } from '@/global/TimeService.js';
import { Packed } from '@/misc/json-schema.js';
function defaultActiveThreshold() {
return new Date(Date.now() - 1000 * 60 * 60 * 24 * 30);
}
@Injectable()
export class UserSearchService {
constructor(
@ -36,9 +33,14 @@ export class UserSearchService {
private mutingsRepository: MutingsRepository,
private userEntityService: UserEntityService,
private readonly timeService: TimeService,
) {
}
private defaultActiveThreshold() {
return new Date(this.timeService.now - 1000 * 60 * 60 * 24 * 30);
}
/**
* ユーザ名とホスト名によるユーザ検索を行う.
*
@ -120,7 +122,7 @@ export class UserSearchService {
},
) {
// デフォルト30日以内に更新されたユーザーをアクティブユーザーとする
const activeThreshold = params.activeThreshold ?? defaultActiveThreshold();
const activeThreshold = params.activeThreshold ?? this.defaultActiveThreshold();
const followingUserQuery = this.followingsRepository.createQueryBuilder('following')
.select('following.followeeId')
@ -166,7 +168,7 @@ export class UserSearchService {
activeThreshold?: Date,
}) {
// デフォルト30日以内に更新されたユーザーをアクティブユーザーとする
const activeThreshold = params.activeThreshold ?? defaultActiveThreshold();
const activeThreshold = params.activeThreshold ?? this.defaultActiveThreshold();
const activeUserQuery = this.generateUserQueryBuilder(params)
.andWhere(new Brackets(qb => {
@ -218,7 +220,7 @@ export class UserSearchService {
offset: number;
origin: 'local' | 'remote' | 'combined';
}> = {}) {
const activeThreshold = new Date(Date.now() - (1000 * 60 * 60 * 24 * 30)); // 30日
const activeThreshold = new Date(this.timeService.now - (1000 * 60 * 60 * 24 * 30)); // 30日
const isUsername = query.startsWith('@') && !query.includes(' ') && query.indexOf('@', 1) === -1;

6
packages/backend/src/core/UserService.ts
View file

@ -11,6 +11,7 @@ import { bindThis } from '@/decorators.js';
import { SystemWebhookService } from '@/core/SystemWebhookService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { CacheService } from '@/core/CacheService.js';
import { TimeService } from '@/global/TimeService.js';
@Injectable()
export class UserService {
@ -22,6 +23,7 @@ export class UserService {
private systemWebhookService: SystemWebhookService,
private userEntityService: UserEntityService,
private readonly cacheService: CacheService,
private readonly timeService: TimeService,
) {
}
@ -30,7 +32,7 @@ export class UserService {
if (user.isHibernated) {
const result = await this.usersRepository.createQueryBuilder().update()
.set({
lastActiveDate: new Date(),
lastActiveDate: this.timeService.date,
})
.where('id = :id', { id: user.id })
.returning('*')
@ -54,7 +56,7 @@ export class UserService {
}
} else {
this.usersRepository.update(user.id, {
lastActiveDate: new Date(),
lastActiveDate: this.timeService.date,
});
}
}

2
packages/backend/src/core/UserSuspendService.ts
View file

@ -21,7 +21,7 @@ import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { trackPromise } from '@/misc/promise-tracker.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
@Injectable()
export class UserSuspendService {

81
packages/backend/src/core/UserWebhookService.ts
View file

@ -9,9 +9,11 @@ import { MiUser, type WebhooksRepository } from '@/models/_.js';
import { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { GlobalEvents } from '@/core/GlobalEventService.js';
import type { InternalEventTypes } from '@/core/GlobalEventService.js';
import type { Packed } from '@/misc/json-schema.js';
import { QueueService } from '@/core/QueueService.js';
import { CacheManagementService, type ManagedMemorySingleCache } from '@/global/CacheManagementService.js';
import { InternalEventService } from '@/global/InternalEventService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
export type UserWebhookPayload<T extends WebhookEventTypes> =
@ -27,8 +29,7 @@ export type UserWebhookPayload<T extends WebhookEventTypes> =
@Injectable()
export class UserWebhookService implements OnApplicationShutdown {
private activeWebhooksFetched = false;
private activeWebhooks: MiWebhook[] = [];
private readonly activeWebhooks: ManagedMemorySingleCache<MiWebhook[]>;
constructor(
@Inject(DI.redisForSub)
@ -36,20 +37,24 @@ export class UserWebhookService implements OnApplicationShutdown {
@Inject(DI.webhooksRepository)
private webhooksRepository: WebhooksRepository,
private queueService: QueueService,
private readonly internalEventService: InternalEventService,
cacheManagementService: CacheManagementService,
) {
this.redisForSub.on('message', this.onMessage);
this.activeWebhooks = cacheManagementService.createMemorySingleCache<MiWebhook[]>('userWebhooks', 1000 * 60 * 60 * 12); // 12h
this.internalEventService.on('webhookCreated', this.onWebhookEvent);
this.internalEventService.on('webhookUpdated', this.onWebhookEvent);
this.internalEventService.on('webhookDeleted', this.onWebhookEvent);
}
@bindThis
public async getActiveWebhooks() {
if (!this.activeWebhooksFetched) {
this.activeWebhooks = await this.webhooksRepository.findBy({
return await this.activeWebhooks.fetch(async () => {
return await this.webhooksRepository.findBy({
active: true,
});
this.activeWebhooksFetched = true;
}
return this.activeWebhooks;
});
}
/**
@ -97,57 +102,51 @@ export class UserWebhookService implements OnApplicationShutdown {
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel !== 'internal') {
private async onWebhookEvent<E extends 'webhookCreated' | 'webhookUpdated' | 'webhookDeleted'>(body: InternalEventTypes[E], type: E): Promise<void> {
const cache = this.activeWebhooks.get();
if (!cache) {
return;
}
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
switch (type) {
case 'webhookCreated': {
if (body.active) {
this.activeWebhooks.push({ // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい
...body,
latestSentAt: body.latestSentAt ? new Date(body.latestSentAt) : null,
user: null, // joinなカラムは通常取ってこないので
});
// Add
const webhook = await this.webhooksRepository.findOneBy({ id: body.id });
if (webhook) {
cache.push(webhook);
}
break;
}
case 'webhookUpdated': {
if (body.active) {
const i = this.activeWebhooks.findIndex(a => a.id === body.id);
if (i > -1) {
this.activeWebhooks[i] = { // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい
...body,
latestSentAt: body.latestSentAt ? new Date(body.latestSentAt) : null,
user: null, // joinなカラムは通常取ってこないので
};
} else {
this.activeWebhooks.push({ // TODO: このあたりのデシリアライズ処理は各modelファイル内に関数としてexportしたい
...body,
latestSentAt: body.latestSentAt ? new Date(body.latestSentAt) : null,
user: null, // joinなカラムは通常取ってこないので
});
}
} else {
this.activeWebhooks = this.activeWebhooks.filter(a => a.id !== body.id);
// Delete
const index = cache.findIndex(webhook => webhook.id === body.id);
if (index > -1) {
cache.splice(index, 1);
}
// Add
const webhook = await this.webhooksRepository.findOneBy({ id: body.id });
if (webhook) {
cache.push(webhook);
}
break;
}
case 'webhookDeleted': {
this.activeWebhooks = this.activeWebhooks.filter(a => a.id !== body.id);
// Delete
const index = cache.findIndex(webhook => webhook.id === body.id);
if (index > -1) {
cache.splice(index, 1);
}
break;
}
default:
break;
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.internalEventService.off('webhookCreated', this.onWebhookEvent);
this.internalEventService.off('webhookUpdated', this.onWebhookEvent);
this.internalEventService.off('webhookDeleted', this.onWebhookEvent);
}
@bindThis

2
packages/backend/src/core/UtilityService.ts
View file

@ -14,7 +14,7 @@ import { bindThis } from '@/decorators.js';
import type { MiMeta, SoftwareSuspension } from '@/models/Meta.js';
import type { MiInstance } from '@/models/Instance.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { EnvService } from '@/core/EnvService.js';
import { EnvService } from '@/global/EnvService.js';
@Injectable()
export class UtilityService {

Some files were not shown because too many files have changed in this diff Show more