fix emoji caching
This commit is contained in:
Hazelnoot
parent
df5aff9d91
commit
bb0925224d
14 changed files with 552 additions and 288 deletions
|
|
@ -820,7 +820,7 @@ export class ChatService {
|
|||
reaction = normalizeEmojiString(reaction_);
|
||||
} else {
|
||||
const name = custom[1];
|
||||
const emoji = (await this.customEmojiService.localEmojisCache.fetch()).get(name);
|
||||
const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(name);
|
||||
|
||||
if (emoji == null) {
|
||||
throw new Error('no such emoji');
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
|
||||
import * as Redis from 'ioredis';
|
||||
import { In, IsNull } from 'typeorm';
|
||||
import { In, IsNull, Not } from 'typeorm';
|
||||
import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js';
|
||||
import { GlobalEventService } from '@/core/GlobalEventService.js';
|
||||
import { IdService } from '@/core/IdService.js';
|
||||
|
|
@ -14,12 +14,29 @@ import { bindThis } from '@/decorators.js';
|
|||
import { DI } from '@/di-symbols.js';
|
||||
import { MemoryKVCache, RedisSingleCache } from '@/misc/cache.js';
|
||||
import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
|
||||
import type { DriveFilesRepository, EmojisRepository, MiRole, MiUser } from '@/models/_.js';
|
||||
import type { DriveFilesRepository, EmojisRepository, MiRole, MiUser, MiDriveFile, NotesRepository } from '@/models/_.js';
|
||||
import type { MiEmoji } from '@/models/Emoji.js';
|
||||
import type { Serialized } from '@/types.js';
|
||||
import { ModerationLogService } from '@/core/ModerationLogService.js';
|
||||
import type { Config } from '@/config.js';
|
||||
import { DriveService } from './DriveService.js';
|
||||
import { DriveService } from '@/core/DriveService.js';
|
||||
import { CacheManagementService, type ManagedQuantumKVCache } from '@/core/CacheManagementService.js';
|
||||
import { TimeService } from '@/core/TimeService.js';
|
||||
import { LoggerService } from '@/core/LoggerService.js';
|
||||
import { isRetryableSymbol } from '@/misc/is-retryable-error.js';
|
||||
import { KeyNotFoundError } from '@/misc/QuantumKVCache.js';
|
||||
import type Logger from '@/logger.js';
|
||||
|
||||
// TODO move to sk-types.d.ts when merged
|
||||
type MinEntity<T> = Omit<T, NullableProps<T>> & {
|
||||
[K in NullableProps<T>]?: T[K] | undefined;
|
||||
};
|
||||
type SemiPartial<T, P extends keyof T> = Omit<T, P> & {
|
||||
[Key in P]?: T[Key] | undefined;
|
||||
};
|
||||
type NullableProps<T> = {
|
||||
[K in keyof T]: null extends T[K] ? K : never;
|
||||
}[keyof T];
|
||||
|
||||
const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/;
|
||||
|
||||
|
|
@ -60,9 +77,15 @@ export const fetchEmojisSortKeys = [
|
|||
export type FetchEmojisSortKeys = typeof fetchEmojisSortKeys[number];
|
||||
|
||||
@Injectable()
|
||||
export class CustomEmojiService implements OnApplicationShutdown {
|
||||
private emojisCache: MemoryKVCache<MiEmoji | null>;
|
||||
public localEmojisCache: RedisSingleCache<Map<string, MiEmoji>>;
|
||||
export class CustomEmojiService {
|
||||
// id -> MiEmoji
|
||||
public readonly emojisByIdCache: ManagedQuantumKVCache<MiEmoji>;
|
||||
|
||||
// key ("name host") -> MiEmoji (for remote emojis)
|
||||
// key ("name") -> MiEmoji (for local emojis)
|
||||
public readonly emojisByKeyCache: ManagedQuantumKVCache<MiEmoji>;
|
||||
|
||||
private readonly logger: Logger;
|
||||
|
||||
constructor(
|
||||
@Inject(DI.redis)
|
||||
|
|
@ -73,29 +96,47 @@ export class CustomEmojiService implements OnApplicationShutdown {
|
|||
private emojisRepository: EmojisRepository,
|
||||
@Inject(DI.driveFilesRepository)
|
||||
private driveFilesRepository: DriveFilesRepository,
|
||||
|
||||
@Inject(DI.notesRepository)
|
||||
private notesRepository: NotesRepository,
|
||||
|
||||
private utilityService: UtilityService,
|
||||
private idService: IdService,
|
||||
private emojiEntityService: EmojiEntityService,
|
||||
private moderationLogService: ModerationLogService,
|
||||
private globalEventService: GlobalEventService,
|
||||
private driveService: DriveService,
|
||||
) {
|
||||
this.emojisCache = new MemoryKVCache<MiEmoji | null>(1000 * 60 * 60 * 12); // 12h
|
||||
private readonly timeService: TimeService,
|
||||
|
||||
this.localEmojisCache = new RedisSingleCache<Map<string, MiEmoji>>(this.redisClient, 'localEmojis', {
|
||||
lifetime: 1000 * 60 * 30, // 30m
|
||||
memoryCacheLifetime: 1000 * 60 * 3, // 3m
|
||||
fetcher: () => this.emojisRepository.find({ where: { host: IsNull() } }).then(emojis => new Map(emojis.map(emoji => [emoji.name, emoji]))),
|
||||
toRedisConverter: (value) => JSON.stringify(Array.from(value.values())),
|
||||
fromRedisConverter: (value) => {
|
||||
return new Map(JSON.parse(value).map((x: Serialized<MiEmoji>) => [x.name, {
|
||||
...x,
|
||||
updatedAt: x.updatedAt ? new Date(x.updatedAt) : null,
|
||||
}]));
|
||||
cacheManagementService: CacheManagementService,
|
||||
loggerService: LoggerService,
|
||||
) {
|
||||
this.logger = loggerService.getLogger('custom-emoji');
|
||||
|
||||
this.emojisByIdCache = cacheManagementService.createQuantumKVCache<MiEmoji>('emojisById', {
|
||||
lifetime: 1000 * 60 * 60, // 1h
|
||||
fetcher: async (id) => await this.emojisRepository.findOneBy({ id }),
|
||||
bulkFetcher: async (ids) => await this.emojisRepository.findBy({ id: In(ids) }).then(es => es.map(e => [e.id, e])),
|
||||
});
|
||||
|
||||
this.emojisByKeyCache = cacheManagementService.createQuantumKVCache<MiEmoji>('emojisByKey', {
|
||||
lifetime: 1000 * 60 * 60, // 1h
|
||||
fetcher: async (key) => {
|
||||
const { host, name } = decodeEmojiKey(key);
|
||||
return await this.emojisRepository.findOneBy({ host: host ?? IsNull(), name });
|
||||
},
|
||||
bulkFetcher: async (keys) => {
|
||||
const queries = keys.map(key => {
|
||||
const { host, name } = decodeEmojiKey(key);
|
||||
return { host: host ?? IsNull(), name };
|
||||
});
|
||||
const emojis = await this.emojisRepository.findBy(queries);
|
||||
return emojis.map(emoji => [encodeEmojiKey(emoji), emoji]);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** @deprecated use createEmoji for new code */
|
||||
@bindThis
|
||||
public async add(data: {
|
||||
originalUrl: string;
|
||||
|
|
@ -110,31 +151,39 @@ export class CustomEmojiService implements OnApplicationShutdown {
|
|||
localOnly: boolean;
|
||||
roleIdsThatCanBeUsedThisEmojiAsReaction: MiRole['id'][];
|
||||
}, moderator?: MiUser): Promise<MiEmoji> {
|
||||
const emoji = await this.emojisRepository.insertOne({
|
||||
id: this.idService.gen(),
|
||||
updatedAt: new Date(),
|
||||
name: data.name,
|
||||
category: data.category,
|
||||
host: data.host,
|
||||
aliases: data.aliases,
|
||||
originalUrl: data.originalUrl,
|
||||
publicUrl: data.publicUrl,
|
||||
type: data.fileType,
|
||||
license: data.license,
|
||||
isSensitive: data.isSensitive,
|
||||
localOnly: data.localOnly,
|
||||
roleIdsThatCanBeUsedThisEmojiAsReaction: data.roleIdsThatCanBeUsedThisEmojiAsReaction,
|
||||
});
|
||||
return await this.createEmoji(data, { moderator });
|
||||
}
|
||||
|
||||
if (data.host == null) {
|
||||
this.localEmojisCache.refresh();
|
||||
public async createEmoji(
|
||||
data: SemiPartial<MinEntity<MiEmoji>, 'id' | 'updatedAt' | 'aliases' | 'roleIdsThatCanBeUsedThisEmojiAsReaction'>,
|
||||
opts?: { moderator?: { id: string } },
|
||||
): Promise<MiEmoji> {
|
||||
// Set defaults
|
||||
data.id ??= this.idService.gen();
|
||||
data.updatedAt ??= this.timeService.date;
|
||||
data.aliases ??= [];
|
||||
data.roleIdsThatCanBeUsedThisEmojiAsReaction ??= [];
|
||||
|
||||
this.globalEventService.publishBroadcastStream('emojiAdded', {
|
||||
emoji: await this.emojiEntityService.packDetailed(emoji.id),
|
||||
// Add to logs
|
||||
this.logger.info(`Creating emoji name=${data.name} host=${data.host}...`);
|
||||
|
||||
// Add to database
|
||||
await this.emojisRepository.insert(data);
|
||||
|
||||
// Add to cache
|
||||
const emoji = await this.emojisByIdCache.fetch(data.id);
|
||||
const emojiKey = encodeEmojiKey({ name: emoji.name, host: emoji.host });
|
||||
this.emojisByIdCache.add(emojiKey, emoji); // This is a new entity, so we can use add() which does not emit sync events.
|
||||
|
||||
if (emoji.host == null) {
|
||||
// Add to clients
|
||||
await this.globalEventService.publishBroadcastStream('emojiAdded', {
|
||||
emoji: await this.emojiEntityService.packDetailed(emoji),
|
||||
});
|
||||
|
||||
if (moderator) {
|
||||
this.moderationLogService.log(moderator, 'addCustomEmoji', {
|
||||
// Add to mod logs
|
||||
if (opts?.moderator) {
|
||||
await this.moderationLogService.log(opts.moderator, 'addCustomEmoji', {
|
||||
emojiId: emoji.id,
|
||||
emoji: emoji,
|
||||
});
|
||||
|
|
@ -144,6 +193,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
|
|||
return emoji;
|
||||
}
|
||||
|
||||
/** @deprecated Use updateEmoji for new code */
|
||||
@bindThis
|
||||
public async update(data: (
|
||||
{ id: MiEmoji['id'], name?: string; } | { name: string; id?: MiEmoji['id'], }
|
||||
|
|
@ -161,91 +211,161 @@ export class CustomEmojiService implements OnApplicationShutdown {
|
|||
null
|
||||
| 'NO_SUCH_EMOJI'
|
||||
| 'SAME_NAME_EMOJI_EXISTS'
|
||||
> {
|
||||
const emoji = data.id
|
||||
? await this.getEmojiById(data.id)
|
||||
: await this.getEmojiByName(data.name!);
|
||||
if (emoji === null) return 'NO_SUCH_EMOJI';
|
||||
const id = emoji.id;
|
||||
> {
|
||||
try {
|
||||
const criteria = data.id
|
||||
? { id: data.id as string }
|
||||
: { name: data.name as string, host: null };
|
||||
|
||||
// IDと絵文字名が両方指定されている場合は絵文字名の変更を行うため重複チェックが必要
|
||||
const doNameUpdate = data.id && data.name && (data.name !== emoji.name);
|
||||
const updates = {
|
||||
...data,
|
||||
id: undefined,
|
||||
host: undefined,
|
||||
};
|
||||
|
||||
const opts = {
|
||||
moderator,
|
||||
};
|
||||
|
||||
await this.updateEmoji(criteria, updates, opts);
|
||||
return null;
|
||||
} catch (err) {
|
||||
if (err instanceof KeyNotFoundError) return 'NO_SUCH_EMOJI';
|
||||
if (err instanceof DuplicateEmojiError) return 'SAME_NAME_EMOJI_EXISTS';
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async updateEmoji(
|
||||
criteria: { id: string } | { name: string, host: string | null },
|
||||
data: Omit<Partial<MiEmoji>, 'id' | 'host'>,
|
||||
opts?: { moderator?: { id: string } },
|
||||
): Promise<MiEmoji> {
|
||||
const emoji = 'id' in criteria
|
||||
? await this.emojisByIdCache.fetch(criteria.id)
|
||||
: await this.emojisByKeyCache.fetch(encodeEmojiKey(criteria));
|
||||
|
||||
// Update the system logs
|
||||
this.logger.info(`Creating emoji name=${emoji.name} host=${emoji.host}...`);
|
||||
|
||||
// If changing the name, then make sure we don't have a conflict.
|
||||
const doNameUpdate = data.name !== undefined && data.name !== emoji.name;
|
||||
if (doNameUpdate) {
|
||||
const isDuplicate = await this.checkDuplicate(data.name!);
|
||||
if (isDuplicate) return 'SAME_NAME_EMOJI_EXISTS';
|
||||
const isDuplicate = await this.checkDuplicate(data.name as string, emoji.host);
|
||||
if (isDuplicate) throw new DuplicateEmojiError(data.name as string, emoji.host);
|
||||
}
|
||||
|
||||
await this.emojisRepository.update(emoji.id, {
|
||||
updatedAt: new Date(),
|
||||
name: data.name,
|
||||
category: data.category,
|
||||
aliases: data.aliases,
|
||||
license: data.license,
|
||||
isSensitive: data.isSensitive,
|
||||
localOnly: data.localOnly,
|
||||
originalUrl: data.originalUrl,
|
||||
publicUrl: data.publicUrl,
|
||||
type: data.fileType,
|
||||
roleIdsThatCanBeUsedThisEmojiAsReaction: data.roleIdsThatCanBeUsedThisEmojiAsReaction ?? undefined,
|
||||
});
|
||||
// Make sure we always set the updated date!
|
||||
data.updatedAt ??= this.timeService.date;
|
||||
|
||||
this.localEmojisCache.refresh();
|
||||
// Update the database
|
||||
await this.emojisRepository.update({ id: emoji.id }, data);
|
||||
|
||||
// If we're changing the file, then we need to delete the old one
|
||||
if (data.originalUrl != null && data.originalUrl !== emoji.originalUrl) {
|
||||
const oldFile = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() });
|
||||
const newFile = await this.driveFilesRepository.findOneBy({ url: data.originalUrl, userHost: emoji.host ? emoji.host : IsNull() });
|
||||
// Update the caches
|
||||
const updated = await this.emojisByIdCache.refresh(emoji.id);
|
||||
const updatedKey = encodeEmojiKey({ name: emoji.name, host: emoji.host });
|
||||
await this.emojisByKeyCache.set(updatedKey, updated);
|
||||
|
||||
// But DON'T delete if this is the same file reference, otherwise we'll break the emoji!
|
||||
if (oldFile && newFile && oldFile.id !== newFile.id) {
|
||||
await this.driveService.deleteFile(oldFile, false, moderator ? moderator : undefined);
|
||||
}
|
||||
// Update the file
|
||||
await this.updateEmojiFile(emoji, updated);
|
||||
|
||||
// If it's a remote emoji, then we're done.
|
||||
// The remaining logic applies only to local emojis.
|
||||
if (updated.host != null) {
|
||||
return updated;
|
||||
}
|
||||
|
||||
const packed = await this.emojiEntityService.packDetailed(emoji.id);
|
||||
|
||||
// Update the clients
|
||||
if (!doNameUpdate) {
|
||||
this.globalEventService.publishBroadcastStream('emojiUpdated', {
|
||||
// If name is the same, then we can update in-place
|
||||
const packed = await this.emojiEntityService.packDetailed(updated);
|
||||
await this.globalEventService.publishBroadcastStream('emojiUpdated', {
|
||||
emojis: [packed],
|
||||
});
|
||||
} else {
|
||||
this.globalEventService.publishBroadcastStream('emojiDeleted', {
|
||||
emojis: [await this.emojiEntityService.packDetailed(emoji)],
|
||||
// If name has changed, we need to delete and recreate
|
||||
const [oldPacked, newPacked] = await Promise.all([
|
||||
this.emojiEntityService.packDetailed(emoji),
|
||||
this.emojiEntityService.packDetailed(updated),
|
||||
]);
|
||||
|
||||
await this.globalEventService.publishBroadcastStream('emojiDeleted', {
|
||||
emojis: [oldPacked],
|
||||
});
|
||||
|
||||
this.globalEventService.publishBroadcastStream('emojiAdded', {
|
||||
emoji: packed,
|
||||
await this.globalEventService.publishBroadcastStream('emojiAdded', {
|
||||
emoji: newPacked,
|
||||
});
|
||||
}
|
||||
|
||||
if (moderator) {
|
||||
const updated = await this.emojisRepository.findOneByOrFail({ id: id });
|
||||
this.moderationLogService.log(moderator, 'updateCustomEmoji', {
|
||||
// Update the mod logs
|
||||
if (opts?.moderator) {
|
||||
await this.moderationLogService.log(opts.moderator, 'updateCustomEmoji', {
|
||||
emojiId: emoji.id,
|
||||
before: emoji,
|
||||
after: updated,
|
||||
});
|
||||
}
|
||||
return null;
|
||||
|
||||
return updated;
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private async updateEmojiFile(before: MiEmoji, after: MiEmoji, moderator?: { id: string }): Promise<void> {
|
||||
// Nothing to do
|
||||
if (after.originalUrl === before.originalUrl) {
|
||||
return;
|
||||
}
|
||||
|
||||
// If we're changing the file, then we need to delete the old one.
|
||||
const [oldFile, newFile] = await Promise.all([
|
||||
this.driveFilesRepository.findOneBy({ url: before.originalUrl, userHost: before.host ?? IsNull() }),
|
||||
this.driveFilesRepository.findOneBy({ url: after.originalUrl, userHost: after.host ?? IsNull() }),
|
||||
]);
|
||||
|
||||
// But DON'T delete if this is the same file reference, otherwise we'll break the emoji!
|
||||
if (!oldFile || !newFile || oldFile.id === newFile.id) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.safeDeleteEmojiFile(before, oldFile, moderator);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private async safeDeleteEmojiFile(emoji: MiEmoji, file: MiDriveFile, moderator?: { id: string }): Promise<void> {
|
||||
const [hasNoteReferences, hasEmojiReferences] = await Promise.all([
|
||||
// Any note using this file ID is a reference.
|
||||
this.notesRepository
|
||||
.createQueryBuilder('note')
|
||||
.where(':fileId <@ note.fileIds', { fileId: file.id })
|
||||
.getExists(),
|
||||
// Any *other* emoji using this file URL is a reference.
|
||||
this.emojisRepository.existsBy({
|
||||
originalUrl: file.url,
|
||||
id: Not(emoji.id),
|
||||
}),
|
||||
]);
|
||||
|
||||
if (hasNoteReferences) {
|
||||
this.logger.debug(`Not removing old file ${file.id} (${file.url}) - file is referenced by one or more notes.`);
|
||||
} else if (hasEmojiReferences) {
|
||||
this.logger.debug(`Not removing old file ${file.id} (${file.url}) - file is reference by another emoji.`);
|
||||
} else {
|
||||
this.logger.info(`Removing old file ${file.id} (${file.url}).`);
|
||||
await this.driveService.deleteFile(file, false, moderator);
|
||||
}
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async addAliasesBulk(ids: MiEmoji['id'][], aliases: string[]) {
|
||||
const emojis = await this.emojisRepository.findBy({
|
||||
id: In(ids),
|
||||
});
|
||||
|
||||
for (const emoji of emojis) {
|
||||
await this.emojisRepository.update(emoji.id, {
|
||||
updatedAt: new Date(),
|
||||
aliases: [...new Set(emoji.aliases.concat(aliases))],
|
||||
});
|
||||
}
|
||||
|
||||
this.localEmojisCache.refresh();
|
||||
|
||||
this.globalEventService.publishBroadcastStream('emojiUpdated', {
|
||||
emojis: await this.emojiEntityService.packDetailedMany(ids),
|
||||
await this.bulkUpdateEmojis(ids, async emojis => {
|
||||
for (const emoji of emojis) {
|
||||
await this.emojisRepository.update(emoji.id, {
|
||||
updatedAt: new Date(),
|
||||
aliases: [...new Set(emoji.aliases.concat(aliases))],
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -258,30 +378,18 @@ export class CustomEmojiService implements OnApplicationShutdown {
|
|||
aliases: aliases,
|
||||
});
|
||||
|
||||
this.localEmojisCache.refresh();
|
||||
|
||||
this.globalEventService.publishBroadcastStream('emojiUpdated', {
|
||||
emojis: await this.emojiEntityService.packDetailedMany(ids),
|
||||
});
|
||||
await this.bulkUpdateEmojis(ids);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async removeAliasesBulk(ids: MiEmoji['id'][], aliases: string[]) {
|
||||
const emojis = await this.emojisRepository.findBy({
|
||||
id: In(ids),
|
||||
});
|
||||
|
||||
for (const emoji of emojis) {
|
||||
await this.emojisRepository.update(emoji.id, {
|
||||
updatedAt: new Date(),
|
||||
aliases: emoji.aliases.filter(x => !aliases.includes(x)),
|
||||
});
|
||||
}
|
||||
|
||||
this.localEmojisCache.refresh();
|
||||
|
||||
this.globalEventService.publishBroadcastStream('emojiUpdated', {
|
||||
emojis: await this.emojiEntityService.packDetailedMany(ids),
|
||||
await this.bulkUpdateEmojis(ids, async emojis => {
|
||||
for (const emoji of emojis) {
|
||||
await this.emojisRepository.update(emoji.id, {
|
||||
updatedAt: new Date(),
|
||||
aliases: emoji.aliases.filter(x => !aliases.includes(x)),
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -294,11 +402,7 @@ export class CustomEmojiService implements OnApplicationShutdown {
|
|||
category: category,
|
||||
});
|
||||
|
||||
this.localEmojisCache.refresh();
|
||||
|
||||
this.globalEventService.publishBroadcastStream('emojiUpdated', {
|
||||
emojis: await this.emojiEntityService.packDetailedMany(ids),
|
||||
});
|
||||
await this.bulkUpdateEmojis(ids);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
|
|
@ -310,67 +414,111 @@ export class CustomEmojiService implements OnApplicationShutdown {
|
|||
license: license,
|
||||
});
|
||||
|
||||
this.localEmojisCache.refresh();
|
||||
await this.bulkUpdateEmojis(ids);
|
||||
}
|
||||
|
||||
this.globalEventService.publishBroadcastStream('emojiUpdated', {
|
||||
emojis: await this.emojiEntityService.packDetailedMany(ids),
|
||||
@bindThis
|
||||
private async bulkUpdateEmojis(ids: MiEmoji['id'][], updater?: (emojis: readonly MiEmoji[]) => Promise<void>): Promise<void> {
|
||||
// Update the database
|
||||
if (updater) {
|
||||
const emojis = await this.emojisByIdCache.fetchMany(ids);
|
||||
await updater(emojis.values);
|
||||
}
|
||||
|
||||
// Update the caches
|
||||
const updated = await this.emojisByIdCache.refreshMany(ids);
|
||||
const keyUpdates = updated.values.map(emoji => [encodeEmojiKey(emoji), emoji] as const);
|
||||
await this.emojisByKeyCache.setMany(keyUpdates);
|
||||
|
||||
// Update the clients
|
||||
await this.globalEventService.publishBroadcastStream('emojiUpdated', {
|
||||
emojis: await this.emojiEntityService.packDetailedMany(updated.values),
|
||||
});
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async delete(id: MiEmoji['id'], moderator?: MiUser) {
|
||||
const emoji = await this.emojisRepository.findOneByOrFail({ id: id });
|
||||
public async delete(id: MiEmoji['id'], moderator?: { id: string }) {
|
||||
const emoji = await this.emojisByIdCache.fetch(id);
|
||||
|
||||
await this.emojisRepository.delete(emoji.id);
|
||||
await Promise.all([
|
||||
this.emojisRepository.delete(emoji.id),
|
||||
this.emojisByIdCache.delete(emoji.id),
|
||||
this.emojisByKeyCache.delete(encodeEmojiKey(emoji)),
|
||||
]);
|
||||
|
||||
this.localEmojisCache.refresh();
|
||||
|
||||
const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() });
|
||||
const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ?? IsNull() });
|
||||
|
||||
if (file) {
|
||||
await this.driveService.deleteFile(file, false, moderator ? moderator : undefined);
|
||||
await this.safeDeleteEmojiFile(emoji, file, moderator);
|
||||
}
|
||||
|
||||
this.globalEventService.publishBroadcastStream('emojiDeleted', {
|
||||
emojis: [await this.emojiEntityService.packDetailed(emoji)],
|
||||
});
|
||||
|
||||
if (moderator) {
|
||||
this.moderationLogService.log(moderator, 'deleteCustomEmoji', {
|
||||
emojiId: emoji.id,
|
||||
emoji: emoji,
|
||||
if (emoji.host == null) {
|
||||
await this.globalEventService.publishBroadcastStream('emojiDeleted', {
|
||||
emojis: [await this.emojiEntityService.packDetailed(emoji)],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async deleteBulk(ids: MiEmoji['id'][], moderator?: MiUser) {
|
||||
const emojis = await this.emojisRepository.findBy({
|
||||
id: In(ids),
|
||||
});
|
||||
|
||||
for (const emoji of emojis) {
|
||||
await this.emojisRepository.delete(emoji.id);
|
||||
|
||||
const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() });
|
||||
|
||||
if (file) {
|
||||
await this.driveService.deleteFile(file, false, moderator ? moderator : undefined);
|
||||
}
|
||||
|
||||
if (moderator) {
|
||||
this.moderationLogService.log(moderator, 'deleteCustomEmoji', {
|
||||
await this.moderationLogService.log(moderator, 'deleteCustomEmoji', {
|
||||
emojiId: emoji.id,
|
||||
emoji: emoji,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.localEmojisCache.refresh();
|
||||
@bindThis
|
||||
public async deleteBulk(ids: MiEmoji['id'][], moderator?: MiUser) {
|
||||
const emojis = await this.emojisByIdCache.fetchMany(ids);
|
||||
|
||||
this.globalEventService.publishBroadcastStream('emojiDeleted', {
|
||||
emojis: await this.emojiEntityService.packDetailedMany(emojis),
|
||||
});
|
||||
const filesQueries = emojis.values.map(emoji => ({
|
||||
url: emoji.originalUrl,
|
||||
userHost: emoji.host ?? IsNull(),
|
||||
}));
|
||||
const files = await this.driveFilesRepository.findBy(filesQueries);
|
||||
|
||||
const emojiFiles = emojis.values
|
||||
.map(emoji => {
|
||||
const file = files.find(file => file.url === emoji.originalUrl && file.userHost === emoji.host);
|
||||
return [emoji, file];
|
||||
})
|
||||
.filter(ef => ef[1] != null) as [MiEmoji, MiDriveFile][];
|
||||
|
||||
const localDeleted = emojis.values.filter(emoji => emoji.host == null);
|
||||
const deletedKeys = emojis.values.map(emoji => encodeEmojiKey(emoji));
|
||||
await Promise.all([
|
||||
// Delete from database
|
||||
this.emojisRepository.delete({ id: In(ids) }),
|
||||
this.emojisByIdCache.deleteMany(ids),
|
||||
|
||||
// Delete from cache
|
||||
this.emojisByKeyCache.deleteMany(deletedKeys),
|
||||
|
||||
// Delete from clients
|
||||
localDeleted.length > 0
|
||||
? this.emojiEntityService.packDetailedMany(localDeleted).then(async packed => {
|
||||
await this.globalEventService.publishBroadcastStream('emojiDeleted', {
|
||||
emojis: packed,
|
||||
});
|
||||
})
|
||||
: null,
|
||||
|
||||
// Delete from mod logs
|
||||
localDeleted.length > 0 && moderator != null
|
||||
? Promise.all(localDeleted.map(async emoji => {
|
||||
await this.moderationLogService.log(moderator, 'deleteCustomEmoji', {
|
||||
emojiId: emoji.id,
|
||||
emoji: emoji,
|
||||
});
|
||||
}))
|
||||
: null,
|
||||
|
||||
// Delete from drive
|
||||
emojiFiles.length > 0
|
||||
? Promise.all(emojiFiles.map(async ([emoji, file]) => {
|
||||
await this.safeDeleteEmojiFile(emoji, file, moderator);
|
||||
}))
|
||||
: null,
|
||||
]);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
|
|
@ -407,18 +555,10 @@ export class CustomEmojiService implements OnApplicationShutdown {
|
|||
*/
|
||||
@bindThis
|
||||
public async populateEmoji(emojiName: string, noteUserHost: string | null): Promise<string | null> {
|
||||
const { name, host } = this.parseEmojiStr(emojiName, noteUserHost);
|
||||
if (name == null) return null;
|
||||
if (host == null) return null;
|
||||
const emojiKey = this.translateEmojiKey(emojiName, noteUserHost);
|
||||
if (emojiKey == null) return null;
|
||||
|
||||
const newHost = host === this.config.host ? null : host;
|
||||
|
||||
const queryOrNull = async () => (await this.emojisRepository.findOneBy({
|
||||
name,
|
||||
host: newHost ?? IsNull(),
|
||||
})) ?? null;
|
||||
|
||||
const emoji = await this.emojisCache.fetch(`${name} ${host}`, queryOrNull);
|
||||
const emoji = await this.emojisByKeyCache.fetchMaybe(emojiKey);
|
||||
|
||||
if (emoji == null) return null;
|
||||
return emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ）
|
||||
|
|
@ -440,47 +580,45 @@ export class CustomEmojiService implements OnApplicationShutdown {
|
|||
return res;
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private translateEmojiKey(emojiName: string, noteUserHost: string | null): string | null {
|
||||
const { name, host } = this.parseEmojiStr(emojiName, noteUserHost);
|
||||
if (name == null) return null;
|
||||
if (host == null) return null;
|
||||
|
||||
const newHost = host === this.config.host ? null : host;
|
||||
return encodeEmojiKey({ name, host: newHost });
|
||||
}
|
||||
|
||||
/**
|
||||
* 与えられた絵文字のリストをデータベースから取得し、キャッシュに追加します
|
||||
*/
|
||||
@bindThis
|
||||
public async prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise<void> {
|
||||
const notCachedEmojis = emojis.filter(emoji => this.emojisCache.get(`${emoji.name} ${emoji.host}`) == null);
|
||||
const emojisQuery: any[] = [];
|
||||
const hosts = new Set(notCachedEmojis.map(e => e.host));
|
||||
for (const host of hosts) {
|
||||
if (host == null) continue;
|
||||
emojisQuery.push({
|
||||
name: In(notCachedEmojis.filter(e => e.host === host).map(e => e.name)),
|
||||
host: host,
|
||||
});
|
||||
}
|
||||
const _emojis = emojisQuery.length > 0 ? await this.emojisRepository.find({
|
||||
where: emojisQuery,
|
||||
select: ['name', 'host', 'originalUrl', 'publicUrl'],
|
||||
}) : [];
|
||||
for (const emoji of _emojis) {
|
||||
this.emojisCache.set(`${emoji.name} ${emoji.host}`, emoji);
|
||||
}
|
||||
const emojiKeys = emojis.map(emoji => encodeEmojiKey(emoji));
|
||||
await this.emojisByKeyCache.fetchMany(emojiKeys);
|
||||
}
|
||||
|
||||
/**
|
||||
* ローカル内の絵文字に重複がないかチェックします
|
||||
* @param name 絵文字名
|
||||
* @param host Emoji hostname
|
||||
*/
|
||||
@bindThis
|
||||
public checkDuplicate(name: string): Promise<boolean> {
|
||||
return this.emojisRepository.exists({ where: { name, host: IsNull() } });
|
||||
public async checkDuplicate(name: string, host: string | null = null): Promise<boolean> {
|
||||
const emoji = await this.getEmojiByName(name, host);
|
||||
return emoji != null;
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public getEmojiById(id: string): Promise<MiEmoji | null> {
|
||||
return this.emojisRepository.findOneBy({ id });
|
||||
public async getEmojiById(id: string): Promise<MiEmoji | null> {
|
||||
return await this.emojisByIdCache.fetchMaybe(id) ?? null;
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public getEmojiByName(name: string): Promise<MiEmoji | null> {
|
||||
return this.emojisRepository.findOneBy({ name, host: IsNull() });
|
||||
public async getEmojiByName(name: string, host: string | null = null): Promise<MiEmoji | null> {
|
||||
const emojiKey = encodeEmojiKey({ name, host });
|
||||
return await this.emojisByKeyCache.fetchMaybe(emojiKey) ?? null;
|
||||
}
|
||||
|
||||
@bindThis
|
||||
|
|
@ -627,14 +765,94 @@ export class CustomEmojiService implements OnApplicationShutdown {
|
|||
allPages: Math.ceil(count / limit),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public dispose(): void {
|
||||
this.emojisCache.dispose();
|
||||
}
|
||||
export class InvalidEmojiError extends Error {
|
||||
public readonly [isRetryableSymbol] = false;
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public onApplicationShutdown(signal?: string | undefined): void {
|
||||
this.dispose();
|
||||
export class InvalidEmojiKeyError extends InvalidEmojiError {
|
||||
constructor(
|
||||
public readonly key: string,
|
||||
message?: string,
|
||||
) {
|
||||
const actualMessage = message
|
||||
? `Invalid emoji key "${key}": ${message}`
|
||||
: `Invalid emoji key "${key}".`;
|
||||
super(actualMessage);
|
||||
}
|
||||
}
|
||||
|
||||
export class InvalidEmojiNameError extends InvalidEmojiError {
|
||||
constructor(
|
||||
public readonly name: string,
|
||||
message?: string,
|
||||
) {
|
||||
const actualMessage = message
|
||||
? `Invalid emoji name "${name}": ${message}`
|
||||
: `Invalid emoji name "${name}".`;
|
||||
super(actualMessage);
|
||||
}
|
||||
}
|
||||
|
||||
export class InvalidEmojiHostError extends InvalidEmojiError {
|
||||
constructor(
|
||||
public readonly host: string | null,
|
||||
message?: string,
|
||||
) {
|
||||
const hostString = host == null ? 'null' : `"${host}"`;
|
||||
const actualMessage = message
|
||||
? `Invalid emoji name ${hostString}: ${message}`
|
||||
: `Invalid emoji name ${hostString}.`;
|
||||
super(actualMessage);
|
||||
}
|
||||
}
|
||||
|
||||
export class DuplicateEmojiError extends InvalidEmojiError {
|
||||
constructor(
|
||||
public readonly name: string,
|
||||
public readonly host: string | null,
|
||||
message?: string,
|
||||
) {
|
||||
const hostString = host == null ? 'null' : `"${host}"`;
|
||||
const actualMessage = message
|
||||
? `Duplicate emoji name "${name}" for host ${hostString}: ${message}`
|
||||
: `Duplicate emoji name "${name}" for host ${hostString}.`;
|
||||
super(actualMessage);
|
||||
}
|
||||
}
|
||||
|
||||
export function isValidEmojiName(name: string): boolean {
|
||||
return name !== '' && !name.includes(' ');
|
||||
}
|
||||
|
||||
export function isValidEmojiHost(host: string): boolean {
|
||||
return host !== '' && !host.includes(' ');
|
||||
}
|
||||
|
||||
// TODO unit tests
|
||||
export function encodeEmojiKey(emoji: { name: string, host: string | null }): string {
|
||||
if (emoji.name === '') throw new InvalidEmojiNameError(emoji.name, 'Name cannot be empty.');
|
||||
if (emoji.name.includes(' ')) throw new InvalidEmojiNameError(emoji.name, 'Name cannot contain a space.');
|
||||
|
||||
// Local emojis are just the name.
|
||||
if (emoji.host == null) {
|
||||
return emoji.name;
|
||||
}
|
||||
|
||||
if (emoji.host === '') throw new InvalidEmojiHostError(emoji.host, 'Host cannot be empty.');
|
||||
if (emoji.host.includes(' ')) throw new InvalidEmojiHostError(emoji.host, 'Host cannot contain a space.');
|
||||
return `${emoji.name} ${emoji.host}`;
|
||||
}
|
||||
|
||||
// TODO unit tests
|
||||
export function decodeEmojiKey(key: string): { name: string, host: string | null } {
|
||||
const match = key.match(/^([^ ]+)(?: ([^ ]+))?$/);
|
||||
if (!match) {
|
||||
throw new InvalidEmojiKeyError(key);
|
||||
}
|
||||
|
||||
const name = match[1];
|
||||
const host = match[2] || null;
|
||||
return { name, host };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -739,7 +739,7 @@ export class DriveService {
|
|||
}
|
||||
|
||||
@bindThis
|
||||
public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: MiUser) {
|
||||
public async deleteFile(file: MiDriveFile, isExpired = false, deleter?: { id: string }) {
|
||||
if (file.storedInternal) {
|
||||
this.deleteLocalFile(file.accessKey!);
|
||||
|
||||
|
|
@ -766,7 +766,7 @@ export class DriveService {
|
|||
}
|
||||
|
||||
@bindThis
|
||||
public async deleteFileSync(file: MiDriveFile, isExpired = false, deleter?: MiUser) {
|
||||
public async deleteFileSync(file: MiDriveFile, isExpired = false, deleter?: { id: string }) {
|
||||
const promises = [];
|
||||
|
||||
if (file.storedInternal) {
|
||||
|
|
@ -797,7 +797,7 @@ export class DriveService {
|
|||
}
|
||||
|
||||
@bindThis
|
||||
private async deletePostProcess(file: MiDriveFile, isExpired = false, deleter?: MiUser) {
|
||||
private async deletePostProcess(file: MiDriveFile, isExpired = false, deleter?: { id: string }) {
|
||||
// リモートファイル期限切れ削除後は直リンクにする
|
||||
if (isExpired && file.userHost !== null && file.uri != null) {
|
||||
this.driveFilesRepository.update(file.id, {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
|
|||
import { bindThis } from '@/decorators.js';
|
||||
import { UtilityService } from '@/core/UtilityService.js';
|
||||
import { UserBlockingService } from '@/core/UserBlockingService.js';
|
||||
import { CustomEmojiService } from '@/core/CustomEmojiService.js';
|
||||
import { CustomEmojiService, encodeEmojiKey } from '@/core/CustomEmojiService.js';
|
||||
import { RoleService } from '@/core/RoleService.js';
|
||||
import { FeaturedService } from '@/core/FeaturedService.js';
|
||||
import { trackPromise } from '@/misc/promise-tracker.js';
|
||||
|
|
@ -144,12 +144,8 @@ export class ReactionService {
|
|||
const reacterHost = this.utilityService.toPunyNullable(user.host);
|
||||
|
||||
const name = custom[1];
|
||||
const emoji = reacterHost == null
|
||||
? (await this.customEmojiService.localEmojisCache.fetch()).get(name)
|
||||
: await this.emojisRepository.findOneBy({
|
||||
host: reacterHost,
|
||||
name,
|
||||
});
|
||||
const emojiKey = encodeEmojiKey({ name, host: reacterHost });
|
||||
const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(emojiKey);
|
||||
|
||||
if (emoji) {
|
||||
if (emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.length === 0 || (await this.roleService.getUserRoles(user.id)).some(r => emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.includes(r.id))) {
|
||||
|
|
@ -256,15 +252,9 @@ export class ReactionService {
|
|||
// カスタム絵文字リアクションだったら絵文字情報も送る
|
||||
const decodedReaction = this.decodeReaction(reaction);
|
||||
|
||||
const customEmoji = decodedReaction.name == null ? null : decodedReaction.host == null
|
||||
? (await this.customEmojiService.localEmojisCache.fetch()).get(decodedReaction.name)
|
||||
: await this.emojisRepository.findOne(
|
||||
{
|
||||
where: {
|
||||
name: decodedReaction.name,
|
||||
host: decodedReaction.host,
|
||||
},
|
||||
});
|
||||
const customEmojiKey = decodedReaction.name == null ? null : encodeEmojiKey({ name: decodedReaction.name, host: decodedReaction.host ?? null });
|
||||
const customEmoji = customEmojiKey == null ? null :
|
||||
await this.customEmojiService.emojisByKeyCache.fetchMaybe(customEmojiKey);
|
||||
|
||||
this.globalEventService.publishNoteStream(note.id, 'reacted', {
|
||||
reaction: decodedReaction.reaction,
|
||||
|
|
|
|||
|
|
@ -359,7 +359,7 @@ export class ApRendererService {
|
|||
|
||||
if (reaction.startsWith(':')) {
|
||||
const name = reaction.replaceAll(':', '');
|
||||
const emoji = (await this.customEmojiService.localEmojisCache.fetch()).get(name);
|
||||
const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(name);
|
||||
|
||||
if (emoji && !emoji.localOnly) object.tag = [this.renderEmoji(emoji)];
|
||||
}
|
||||
|
|
@ -948,12 +948,10 @@ export class ApRendererService {
|
|||
}
|
||||
|
||||
@bindThis
|
||||
private async getEmojis(names: string[]): Promise<MiEmoji[]> {
|
||||
private async getEmojis(names: string[]): Promise<readonly MiEmoji[]> {
|
||||
if (names.length === 0) return [];
|
||||
|
||||
const allEmojis = await this.customEmojiService.localEmojisCache.fetch();
|
||||
const emojis = names.map(name => allEmojis.get(name)).filter(x => x != null);
|
||||
|
||||
return emojis;
|
||||
const emojis = await this.customEmojiService.emojisByKeyCache.fetchMany(names);
|
||||
return emojis.values;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,9 @@ import { renderInlineError } from '@/misc/render-inline-error.js';
|
|||
import { extractMediaFromHtml } from '@/core/activitypub/misc/extract-media-from-html.js';
|
||||
import { extractMediaFromMfm } from '@/core/activitypub/misc/extract-media-from-mfm.js';
|
||||
import { getContentByType } from '@/core/activitypub/misc/get-content-by-type.js';
|
||||
import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument, isLink } from '../type.js';
|
||||
import { CustomEmojiService, encodeEmojiKey, isValidEmojiName } from '@/core/CustomEmojiService.js';
|
||||
import { fromTuple } from '@/misc/from-tuple.js';
|
||||
import { getOneApId, getApId, isEmoji, getApType, isApObject, isDocument, IApDocument, isLink, isQuestion, getNullableApId, isPost } from '../type.js';
|
||||
import { ApLoggerService } from '../ApLoggerService.js';
|
||||
import { ApMfmService } from '../ApMfmService.js';
|
||||
import { ApDbResolverService } from '../ApDbResolverService.js';
|
||||
|
|
@ -44,7 +46,7 @@ import { ApMentionService } from './ApMentionService.js';
|
|||
import { ApQuestionService } from './ApQuestionService.js';
|
||||
import { ApImageService } from './ApImageService.js';
|
||||
import type { Resolver } from '../ApResolverService.js';
|
||||
import type { IObject, IPost } from '../type.js';
|
||||
import type { IObject, IPost, IApEmoji } from '../type.js';
|
||||
|
||||
@Injectable()
|
||||
export class ApNoteService {
|
||||
|
|
@ -89,6 +91,7 @@ export class ApNoteService {
|
|||
private apDbResolverService: ApDbResolverService,
|
||||
private apLoggerService: ApLoggerService,
|
||||
private readonly apUtilityService: ApUtilityService,
|
||||
private readonly customEmojiService: CustomEmojiService,
|
||||
) {
|
||||
this.logger = this.apLoggerService.logger;
|
||||
}
|
||||
|
|
@ -563,26 +566,32 @@ export class ApNoteService {
|
|||
// eslint-disable-next-line no-param-reassign
|
||||
host = this.utilityService.toPuny(host);
|
||||
|
||||
const eomjiTags = toArray(tags).filter(isEmoji);
|
||||
const eomjiTags: (IApEmoji & { name: string })[] = toArray(tags)
|
||||
.filter(tag => isEmoji(tag))
|
||||
.map(tag => ({
|
||||
...tag,
|
||||
name: tag.name.replaceAll(':', ''),
|
||||
}))
|
||||
.filter(tag => isValidEmojiName(tag.name));
|
||||
|
||||
const existingEmojis = await this.emojisRepository.findBy({
|
||||
host,
|
||||
name: In(eomjiTags.map(tag => tag.name.replaceAll(':', ''))),
|
||||
});
|
||||
const emojiKeys = eomjiTags.map(tag => encodeEmojiKey({ name: tag.name, host }));
|
||||
const existingEmojis = await this.customEmojiService.emojisByKeyCache.fetchMany(emojiKeys);
|
||||
|
||||
return await Promise.all(eomjiTags.map(async tag => {
|
||||
const name = tag.name.replaceAll(':', '');
|
||||
const name = tag.name;
|
||||
tag.icon = toSingle(tag.icon);
|
||||
|
||||
const exists = existingEmojis.find(x => x.name === name);
|
||||
const exists = existingEmojis.values.find(x => x.name === name);
|
||||
|
||||
if (exists) {
|
||||
if ((exists.updatedAt == null)
|
||||
|| (tag.id != null && exists.uri == null)
|
||||
|| (new Date(tag.updated) > exists.updatedAt)
|
||||
|| (tag.id != null && exists.uri == null) // TODO should we check for ID changes?
|
||||
|| (new Date(tag.updated) > exists.updatedAt) // TODO make sure tag.updated actually exists
|
||||
|| (tag.icon.url !== exists.originalUrl)
|
||||
// TODO check for license changes
|
||||
// TODO check for sensitive changes
|
||||
) {
|
||||
await this.emojisRepository.update({
|
||||
return await this.customEmojiService.updateEmoji({
|
||||
host,
|
||||
name,
|
||||
}, {
|
||||
|
|
@ -593,18 +602,12 @@ export class ApNoteService {
|
|||
// _misskey_license が存在しなければ `null`
|
||||
license: (tag._misskey_license?.freeText ?? null),
|
||||
});
|
||||
|
||||
const emoji = await this.emojisRepository.findOneBy({ host, name });
|
||||
if (emoji == null) throw new Error(`emoji update failed: ${name}:${host}`);
|
||||
return emoji;
|
||||
}
|
||||
|
||||
return exists;
|
||||
}
|
||||
|
||||
this.logger.info(`register emoji host=${host}, name=${name}`);
|
||||
|
||||
return await this.emojisRepository.insertOne({
|
||||
return await this.customEmojiService.createEmoji({
|
||||
id: this.idService.gen(),
|
||||
host,
|
||||
name,
|
||||
|
|
@ -613,8 +616,10 @@ export class ApNoteService {
|
|||
publicUrl: tag.icon.url,
|
||||
updatedAt: new Date(),
|
||||
aliases: [],
|
||||
localOnly: false,
|
||||
isSensitive: tag.sensitive === true,
|
||||
// _misskey_license が存在しなければ `null`
|
||||
license: (tag._misskey_license?.freeText ?? null)
|
||||
license: (tag._misskey_license?.freeText ?? null),
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,17 +3,23 @@
|
|||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { ModuleRef } from '@nestjs/core';
|
||||
import { In } from 'typeorm';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import type { EmojisRepository, MiRole, RolesRepository } from '@/models/_.js';
|
||||
import type { Packed } from '@/misc/json-schema.js';
|
||||
import type { MiEmoji } from '@/models/Emoji.js';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
import type { CustomEmojiService } from '@/core/CustomEmojiService.js';
|
||||
|
||||
@Injectable()
|
||||
export class EmojiEntityService {
|
||||
export class EmojiEntityService implements OnModuleInit {
|
||||
private customEmojiService: CustomEmojiService;
|
||||
|
||||
constructor(
|
||||
private readonly moduleRef: ModuleRef,
|
||||
|
||||
@Inject(DI.emojisRepository)
|
||||
private emojisRepository: EmojisRepository,
|
||||
@Inject(DI.rolesRepository)
|
||||
|
|
@ -21,11 +27,16 @@ export class EmojiEntityService {
|
|||
) {
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public onModuleInit(): void {
|
||||
this.customEmojiService = this.moduleRef.get('CustomEmojiService');
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async packSimple(
|
||||
src: MiEmoji['id'] | MiEmoji,
|
||||
): Promise<Packed<'EmojiSimple'>> {
|
||||
const emoji = typeof src === 'object' ? src : await this.emojisRepository.findOneByOrFail({ id: src });
|
||||
const emoji = typeof src === 'object' ? src : await this.customEmojiService.emojisByIdCache.fetch(src);
|
||||
|
||||
return {
|
||||
aliases: emoji.aliases,
|
||||
|
|
@ -40,17 +51,24 @@ export class EmojiEntityService {
|
|||
}
|
||||
|
||||
@bindThis
|
||||
public packSimpleMany(
|
||||
emojis: any[],
|
||||
public async packSimpleMany(
|
||||
emojis: readonly (MiEmoji | MiEmoji['id'])[],
|
||||
) {
|
||||
return Promise.all(emojis.map(x => this.packSimple(x)));
|
||||
const toFetch = emojis.filter(emoji => typeof(emoji) === 'string');
|
||||
const fetched = new Map(await this.customEmojiService.emojisByIdCache.fetchMany(toFetch));
|
||||
return Promise.all(emojis.map(async x => {
|
||||
if (typeof(x) === 'string') {
|
||||
x = fetched.get(x) ?? await this.customEmojiService.emojisByIdCache.fetch(x);
|
||||
}
|
||||
return this.packSimple(x);
|
||||
}));
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async packDetailed(
|
||||
src: MiEmoji['id'] | MiEmoji,
|
||||
): Promise<Packed<'EmojiDetailed'>> {
|
||||
const emoji = typeof src === 'object' ? src : await this.emojisRepository.findOneByOrFail({ id: src });
|
||||
const emoji = typeof src === 'object' ? src : await this.customEmojiService.emojisByIdCache.fetch(src);
|
||||
|
||||
return {
|
||||
id: emoji.id,
|
||||
|
|
@ -68,10 +86,17 @@ export class EmojiEntityService {
|
|||
}
|
||||
|
||||
@bindThis
|
||||
public packDetailedMany(
|
||||
emojis: any[],
|
||||
public async packDetailedMany(
|
||||
emojis: readonly (MiEmoji | MiEmoji['id'])[],
|
||||
): Promise<Packed<'EmojiDetailed'>[]> {
|
||||
return Promise.all(emojis.map(x => this.packDetailed(x)));
|
||||
const toFetch = emojis.filter(emoji => typeof(emoji) === 'string');
|
||||
const fetched = new Map(await this.customEmojiService.emojisByIdCache.fetchMany(toFetch));
|
||||
return Promise.all(emojis.map(async x => {
|
||||
if (typeof(x) === 'string') {
|
||||
x = fetched.get(x) ?? await this.customEmojiService.emojisByIdCache.fetch(x);
|
||||
}
|
||||
return this.packDetailed(x);
|
||||
}));
|
||||
}
|
||||
|
||||
@bindThis
|
||||
|
|
@ -81,7 +106,7 @@ export class EmojiEntityService {
|
|||
roles?: Map<MiRole['id'], MiRole>
|
||||
},
|
||||
): Promise<Packed<'EmojiDetailedAdmin'>> {
|
||||
const emoji = typeof src === 'object' ? src : await this.emojisRepository.findOneByOrFail({ id: src });
|
||||
const emoji = typeof src === 'object' ? src : await this.customEmojiService.emojisByIdCache.fetch(src);
|
||||
|
||||
const roles = Array.of<MiRole>();
|
||||
if (emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.length > 0) {
|
||||
|
|
@ -136,7 +161,8 @@ export class EmojiEntityService {
|
|||
const emojiEntities = emojis.filter(x => typeof x === 'object') as MiEmoji[];
|
||||
const emojiIdOnlyList = emojis.filter(x => typeof x === 'string') as string[];
|
||||
if (emojiIdOnlyList.length > 0) {
|
||||
emojiEntities.push(...await this.emojisRepository.findBy({ id: In(emojiIdOnlyList) }));
|
||||
const fetched = await this.customEmojiService.emojisByIdCache.fetchMany(emojiIdOnlyList);
|
||||
emojiEntities.push(...fetched.values);
|
||||
}
|
||||
|
||||
// 特定ロール専用の絵文字である場合、そのロール情報をあらかじめまとめて取得しておく（pack側で都度取得も出来るが負荷が高いので）
|
||||
|
|
|
|||
|
|
@ -92,10 +92,11 @@ export class ImportCustomEmojisProcessorService {
|
|||
continue;
|
||||
}
|
||||
const emojiPath = outputPath + '/' + record.fileName;
|
||||
await this.emojisRepository.delete({
|
||||
name: nameNfc,
|
||||
host: IsNull(),
|
||||
});
|
||||
|
||||
const existing = await this.customEmojiService.emojisByIdCache.fetchMaybe(nameNfc);
|
||||
if (existing) {
|
||||
await this.customEmojiService.delete(existing.id, job.data.user);
|
||||
}
|
||||
|
||||
try {
|
||||
const driveFile = await this.driveService.addFile({
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import { IActivity, IAnnounce, ICreate } from '@/core/activitypub/type.js';
|
|||
import { isPureRenote, isQuote, isRenote } from '@/misc/is-renote.js';
|
||||
import * as Acct from '@/misc/acct.js';
|
||||
import { CacheService } from '@/core/CacheService.js';
|
||||
import { CustomEmojiService, encodeEmojiKey } from '@/core/CustomEmojiService.js';
|
||||
import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions, FastifyBodyParser } from 'fastify';
|
||||
import type { FindOptionsWhere } from 'typeorm';
|
||||
import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js';
|
||||
|
|
@ -89,6 +90,7 @@ export class ActivityPubServerService {
|
|||
private fanoutTimelineEndpointService: FanoutTimelineEndpointService,
|
||||
private loggerService: LoggerService,
|
||||
private readonly cacheService: CacheService,
|
||||
private readonly customEmojiService: CustomEmojiService,
|
||||
) {
|
||||
//this.createServer = this.createServer.bind(this);
|
||||
this.logger = this.loggerService.getLogger('apserv', 'pink');
|
||||
|
|
@ -1037,10 +1039,8 @@ export class ActivityPubServerService {
|
|||
const { reject } = await this.checkAuthorizedFetch(request, reply);
|
||||
if (reject) return;
|
||||
|
||||
const emoji = await this.emojisRepository.findOneBy({
|
||||
host: IsNull(),
|
||||
name: request.params.emoji,
|
||||
});
|
||||
const emojiKey = encodeEmojiKey({ name: request.params.emoji, host: null });
|
||||
const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(emojiKey);
|
||||
|
||||
if (emoji == null || emoji.localOnly) {
|
||||
reply.code(404);
|
||||
|
|
@ -1048,7 +1048,7 @@ export class ActivityPubServerService {
|
|||
}
|
||||
|
||||
this.setResponseType(request, reply);
|
||||
return (this.apRendererService.addContext(await this.apRendererService.renderEmoji(emoji)));
|
||||
return (this.apRendererService.addContext(this.apRendererService.renderEmoji(emoji)));
|
||||
});
|
||||
|
||||
// like
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import type Logger from '@/logger.js';
|
|||
import * as Acct from '@/misc/acct.js';
|
||||
import { genIdenticon } from '@/misc/gen-identicon.js';
|
||||
import { UserEntityService } from '@/core/entities/UserEntityService.js';
|
||||
import { CustomEmojiService, encodeEmojiKey } from '@/core/CustomEmojiService.js';
|
||||
import { LoggerService } from '@/core/LoggerService.js';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
import { renderInlineError } from '@/misc/render-inline-error.js';
|
||||
|
|
@ -71,6 +72,7 @@ export class ServerService implements OnApplicationShutdown {
|
|||
private globalEventService: GlobalEventService,
|
||||
private loggerService: LoggerService,
|
||||
private oauth2ProviderService: OAuth2ProviderService,
|
||||
private readonly customEmojiService: CustomEmojiService,
|
||||
) {
|
||||
this.logger = this.loggerService.getLogger('server', 'gray');
|
||||
}
|
||||
|
|
@ -171,14 +173,15 @@ export class ServerService implements OnApplicationShutdown {
|
|||
return;
|
||||
}
|
||||
|
||||
const name = pathChunks.shift();
|
||||
const name = pathChunks.shift() as string;
|
||||
const host = pathChunks.pop();
|
||||
|
||||
const emoji = await this.emojisRepository.findOneBy({
|
||||
const emojiKey = encodeEmojiKey({
|
||||
// `@.` is the spec of ReactionService.decodeReaction
|
||||
host: (host === undefined || host === '.') ? IsNull() : host,
|
||||
host: (host === undefined || host === '.') ? null : host,
|
||||
name: name,
|
||||
});
|
||||
const emoji = await this.customEmojiService.emojisByKeyCache.fetchMaybe(emojiKey);
|
||||
|
||||
reply.header('Content-Security-Policy', 'default-src \'none\'; style-src \'unsafe-inline\'');
|
||||
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
|
|||
private driveService: DriveService,
|
||||
) {
|
||||
super(meta, paramDef, async (ps, me) => {
|
||||
const emoji = await this.emojisRepository.findOneBy({ id: ps.emojiId });
|
||||
const emoji = await this.customEmojiService.emojisByIdCache.fetchMaybe(ps.emojiId);
|
||||
if (emoji == null) {
|
||||
throw new ApiError(meta.errors.noSuchEmoji);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,6 +98,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
|
|||
}
|
||||
|
||||
if (ps.query) {
|
||||
// TODO use string inclusion func instead of dynamic query building
|
||||
q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' })
|
||||
.orderBy('length(emoji.name)', 'ASC');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import type { EmojisRepository } from '@/models/_.js';
|
|||
import { Endpoint } from '@/server/api/endpoint-base.js';
|
||||
import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import { CustomEmojiService } from '@/core/CustomEmojiService.js';
|
||||
|
||||
export const meta = {
|
||||
tags: ['meta'],
|
||||
|
|
@ -47,14 +48,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
|
|||
private emojisRepository: EmojisRepository,
|
||||
|
||||
private emojiEntityService: EmojiEntityService,
|
||||
private readonly customEmojiService: CustomEmojiService,
|
||||
) {
|
||||
super(meta, paramDef, async (ps, me) => {
|
||||
const emoji = await this.emojisRepository.findOneOrFail({
|
||||
where: {
|
||||
name: ps.name,
|
||||
host: IsNull(),
|
||||
},
|
||||
});
|
||||
const emoji = await this.customEmojiService.emojisByKeyCache.fetch(ps.name);
|
||||
|
||||
return this.emojiEntityService.packDetailed(emoji);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@
|
|||
*/
|
||||
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import type { EmojisRepository } from '@/models/_.js';
|
||||
import type { EmojisRepository, MiEmoji } from '@/models/_.js';
|
||||
import { Endpoint } from '@/server/api/endpoint-base.js';
|
||||
import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import { CacheManagementService, type ManagedMemorySingleCache } from '@/core/CacheManagementService.js';
|
||||
import { CustomEmojiService } from '@/core/CustomEmojiService.js';
|
||||
|
||||
export const meta = {
|
||||
tags: ['meta'],
|
||||
|
|
@ -32,8 +34,11 @@ export const meta = {
|
|||
},
|
||||
},
|
||||
|
||||
// 2 calls per second
|
||||
// Up to 20 calls, then 5 / second
|
||||
limit: {
|
||||
type: 'bucket',
|
||||
size: 20,
|
||||
|
||||
duration: 1000,
|
||||
max: 2,
|
||||
},
|
||||
|
|
@ -48,21 +53,41 @@ export const paramDef = {
|
|||
|
||||
@Injectable()
|
||||
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
|
||||
// Short (2 second) cache to handle rapid bursts of fetching the emoji list.
|
||||
// This just stores the IDs - the actual emojis are cached by CustomEmojiService
|
||||
private readonly localEmojiIdsCache: ManagedMemorySingleCache<MiEmoji['id'][]>;
|
||||
|
||||
constructor(
|
||||
@Inject(DI.emojisRepository)
|
||||
private emojisRepository: EmojisRepository,
|
||||
|
||||
private emojiEntityService: EmojiEntityService,
|
||||
private readonly customEmojiService: CustomEmojiService,
|
||||
|
||||
cacheManagementService: CacheManagementService,
|
||||
) {
|
||||
super(meta, paramDef, async (ps, me) => {
|
||||
const emojis = await this.emojisRepository.createQueryBuilder()
|
||||
.where('host IS NULL')
|
||||
.orderBy('LOWER(category)', 'ASC')
|
||||
.addOrderBy('LOWER(name)', 'ASC')
|
||||
.getMany();
|
||||
// Fetch the latest emoji list
|
||||
const emojiIds = await this.localEmojiIdsCache.fetch(async () => {
|
||||
const emojis = await this.emojisRepository.createQueryBuilder()
|
||||
.select('id')
|
||||
.where('host IS NULL')
|
||||
.orderBy('LOWER(category)', 'ASC')
|
||||
.addOrderBy('LOWER(name)', 'ASC')
|
||||
.getMany() as { id: MiEmoji['id'] }[];
|
||||
|
||||
return emojis.map(e => e.id);
|
||||
});
|
||||
|
||||
// Fetch the latest version of each emoji
|
||||
const emojis = await this.customEmojiService.emojisByIdCache.fetchMany(emojiIds);
|
||||
|
||||
// Pack and return everything
|
||||
return {
|
||||
emojis: await this.emojiEntityService.packSimpleMany(emojis),
|
||||
emojis: await this.emojiEntityService.packSimpleMany(emojis.values),
|
||||
};
|
||||
});
|
||||
|
||||
this.localEmojiIdsCache = cacheManagementService.createMemorySingleCache<MiEmoji['id'][]>(1000 * 2);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue