misty.cafe/mistykey
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

merge develop and fix conflicts.

Browse source
This commit is contained in:
piuvas 2025-06-03 10:56:10 -03:00
commit 1120ad19ae
No known key found for this signature in database
166 changed files with 2933 additions and 1079 deletions
Download patch file Download diff file Expand all files Collapse all files

8
packages/backend/src/core/DriveService.ts
View file

@ -159,6 +159,14 @@ export class DriveService {
// thunbnail, webpublic を必要なら生成
const alts = await this.generateAlts(path, type, !file.uri);
if (type && type.startsWith('video/')) {
try {
await this.videoProcessingService.webOptimizeVideo(path, type);
} catch (err) {
this.registerLogger.warn(`Video optimization failed: ${err instanceof Error ? err.message : String(err)}`, { error: err });
}
}
if (this.meta.useObjectStorage) {
//#region ObjectStorage params
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) ?? ['']);

11
packages/backend/src/core/FanoutTimelineEndpointService.ts
View file

@ -136,10 +136,10 @@ export class FanoutTimelineEndpointService {
const parentFilter = filter;
filter = (note) => {
if (!ps.ignoreAuthorFromInstanceBlock) {
if (this.utilityService.isBlockedHost(this.meta.blockedHosts, note.userHost)) return false;
if (note.userInstance?.isBlocked) return false;
}
if (note.userId !== note.renoteUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.renoteUserHost)) return false;
if (note.userId !== note.replyUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.replyUserHost)) return false;
if (note.userId !== note.renoteUserId && note.renoteUserInstance?.isBlocked) return false;
if (note.userId !== note.replyUserId && note.replyUserInstance?.isBlocked) return false;
return parentFilter(note);
};
@ -194,7 +194,10 @@ export class FanoutTimelineEndpointService {
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
.leftJoinAndSelect('note.channel', 'channel')
.leftJoinAndSelect('note.userInstance', 'userInstance')
.leftJoinAndSelect('note.replyUserInstance', 'replyUserInstance')
.leftJoinAndSelect('note.renoteUserInstance', 'renoteUserInstance');
const notes = (await query.getMany()).filter(noteFilter);

103
packages/backend/src/core/FederatedInstanceService.ts
View file

@ -5,23 +5,24 @@
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import { QueryFailedError } from 'typeorm';
import type { InstancesRepository } from '@/models/_.js';
import type { InstancesRepository, MiMeta } from '@/models/_.js';
import type { MiInstance } from '@/models/Instance.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import { MemoryKVCache } from '@/misc/cache.js';
import { IdService } from '@/core/IdService.js';
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { Serialized } from '@/types.js';
import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js';
@Injectable()
export class FederatedInstanceService implements OnApplicationShutdown {
public federatedInstanceCache: RedisKVCache<MiInstance | null>;
private readonly federatedInstanceCache: MemoryKVCache<MiInstance | null>;
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.redisForSub)
private redisForSub: Redis.Redis,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
@ -29,67 +30,46 @@ export class FederatedInstanceService implements OnApplicationShutdown {
private utilityService: UtilityService,
private idService: IdService,
) {
this.federatedInstanceCache = new RedisKVCache<MiInstance | null>(this.redisClient, 'federatedInstance', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60 * 3, // 3m
fetcher: (key) => this.instancesRepository.findOneBy({ host: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => {
const parsed = JSON.parse(value);
if (parsed == null) return null;
return {
...parsed,
firstRetrievedAt: new Date(parsed.firstRetrievedAt),
latestRequestReceivedAt: parsed.latestRequestReceivedAt ? new Date(parsed.latestRequestReceivedAt) : null,
infoUpdatedAt: parsed.infoUpdatedAt ? new Date(parsed.infoUpdatedAt) : null,
notRespondingSince: parsed.notRespondingSince ? new Date(parsed.notRespondingSince) : null,
};
},
});
this.federatedInstanceCache = new MemoryKVCache(1000 * 60 * 3); // 3m
this.redisForSub.on('message', this.onMessage);
}
@bindThis
public async fetchOrRegister(host: string): Promise<MiInstance> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached) return cached;
const index = await this.instancesRepository.findOneBy({ host });
let index = await this.instancesRepository.findOneBy({ host });
if (index == null) {
let i;
try {
i = await this.instancesRepository.insertOne({
await this.instancesRepository.createQueryBuilder('instance')
.insert()
.values({
id: this.idService.gen(),
host,
firstRetrievedAt: new Date(),
});
} catch (e: unknown) {
if (e instanceof QueryFailedError) {
if (isDuplicateKeyValueError(e)) {
i = await this.instancesRepository.findOneBy({ host });
}
}
isBlocked: this.utilityService.isBlockedHost(host),
isSilenced: this.utilityService.isSilencedHost(host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(host),
isAllowListed: this.utilityService.isAllowListedHost(host),
isBubbled: this.utilityService.isBubbledHost(host),
})
.orIgnore()
.execute();
if (i == null) {
throw e;
}
}
this.federatedInstanceCache.set(host, i);
return i;
} else {
this.federatedInstanceCache.set(host, index);
return index;
index = await this.instancesRepository.findOneByOrFail({ host });
}
this.federatedInstanceCache.set(host, index);
return index;
}
@bindThis
public async fetch(host: string): Promise<MiInstance | null> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached !== undefined) return cached;
const index = await this.instancesRepository.findOneBy({ host });
@ -117,8 +97,35 @@ export class FederatedInstanceService implements OnApplicationShutdown {
this.federatedInstanceCache.set(result.host, result);
}
private syncCache(before: Serialized<MiMeta | undefined>, after: Serialized<MiMeta>): void {
const changed =
diffArraysSimple(before?.blockedHosts, after.blockedHosts) ||
diffArraysSimple(before?.silencedHosts, after.silencedHosts) ||
diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) ||
diffArraysSimple(before?.federationHosts, after.federationHosts) ||
diffArraysSimple(before?.bubbleInstances, after.bubbleInstances);
if (changed) {
// We have to clear the whole thing, otherwise subdomains won't be synced.
this.federatedInstanceCache.clear();
}
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
if (type === 'metaUpdated') {
this.syncCache(body.before, body.after);
}
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.federatedInstanceCache.dispose();
}

10
packages/backend/src/core/HttpRequestService.ts
View file

@ -235,7 +235,9 @@ export class HttpRequestService {
}
@bindThis
public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise<IObjectWithId> {
public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const res = await this.send(url, {
method: 'GET',
headers: {
@ -253,7 +255,11 @@ export class HttpRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity as IObjectWithId;
}

11
packages/backend/src/core/InternalStorageService.ts
View file

@ -6,18 +6,11 @@
import * as fs from 'node:fs';
import { copyFile, unlink, writeFile, chmod } from 'node:fs/promises';
import * as Path from 'node:path';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
const path = Path.resolve(_dirname, '../../../../files');
@Injectable()
export class InternalStorageService {
constructor(
@ -25,12 +18,12 @@ export class InternalStorageService {
private config: Config,
) {
// No one should erase the working directory *while the server is running*.
fs.mkdirSync(path, { recursive: true });
fs.mkdirSync(this.config.mediaDirectory, { recursive: true });
}
@bindThis
public resolvePath(key: string) {
return Path.resolve(path, key);
return Path.resolve(this.config.mediaDirectory, key);
}
@bindThis

113
packages/backend/src/core/MetaService.ts
View file

@ -4,7 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { DataSource, EntityManager } from 'typeorm';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { MiMeta } from '@/models/Meta.js';
@ -12,6 +12,9 @@ import { GlobalEventService } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { MiInstance } from '@/models/Instance.js';
import { diffArrays } from '@/misc/diff-arrays.js';
import type { MetasRepository } from '@/models/_.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
@ -26,6 +29,9 @@ export class MetaService implements OnApplicationShutdown {
@Inject(DI.db)
private db: DataSource,
@Inject(DI.metasRepository)
private readonly metasRepository: MetasRepository,
private featuredService: FeaturedService,
private globalEventService: GlobalEventService,
) {
@ -67,35 +73,35 @@ export class MetaService implements OnApplicationShutdown {
public async fetch(noCache = false): Promise<MiMeta> {
if (!noCache && this.cache) return this.cache;
return await this.db.transaction(async transactionalEntityManager => {
// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
const metas = await transactionalEntityManager.find(MiMeta, {
order: {
// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
let meta = await this.metasRepository.createQueryBuilder('meta')
.select()
.orderBy({
id: 'DESC',
})
.limit(1)
.getOne();
if (!meta) {
await this.metasRepository.createQueryBuilder('meta')
.insert()
.values({
id: 'x',
})
.orIgnore()
.execute();
meta = await this.metasRepository.createQueryBuilder('meta')
.select()
.orderBy({
id: 'DESC',
},
});
})
.limit(1)
.getOneOrFail();
}
const meta = metas[0];
if (meta) {
this.cache = meta;
return meta;
} else {
// metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う
const saved = await transactionalEntityManager
.upsert(
MiMeta,
{
id: 'x',
},
['id'],
)
.then((x) => transactionalEntityManager.findOneByOrFail(MiMeta, x.identifiers[0]));
this.cache = saved;
return saved;
}
});
this.cache = meta;
return meta;
}
@bindThis
@ -103,7 +109,7 @@ export class MetaService implements OnApplicationShutdown {
let before: MiMeta | undefined;
const updated = await this.db.transaction(async transactionalEntityManager => {
const metas = await transactionalEntityManager.find(MiMeta, {
const metas: (MiMeta | undefined)[] = await transactionalEntityManager.find(MiMeta, {
order: {
id: 'DESC',
},
@ -126,6 +132,10 @@ export class MetaService implements OnApplicationShutdown {
},
});
// Propagate changes to blockedHosts, silencedHosts, mediaSilencedHosts, federationInstances, and bubbleInstances to the relevant instance rows
// Do this inside the transaction to avoid potential race condition (when an instance gets registered while we're updating).
await this.persistBlocks(transactionalEntityManager, before ?? {}, afters[0]);
return afters[0];
});
@ -159,4 +169,49 @@ export class MetaService implements OnApplicationShutdown {
public onApplicationShutdown(signal?: string | undefined): void {
this.dispose();
}
private async persistBlocks(tem: EntityManager, before: Partial<MiMeta>, after: Partial<MiMeta>): Promise<void> {
await this.persistBlock(tem, before.blockedHosts, after.blockedHosts, 'isBlocked');
await this.persistBlock(tem, before.silencedHosts, after.silencedHosts, 'isSilenced');
await this.persistBlock(tem, before.mediaSilencedHosts, after.mediaSilencedHosts, 'isMediaSilenced');
await this.persistBlock(tem, before.federationHosts, after.federationHosts, 'isAllowListed');
await this.persistBlock(tem, before.bubbleInstances, after.bubbleInstances, 'isBubbled');
}
private async persistBlock(tem: EntityManager, before: string[] | undefined, after: string[] | undefined, field: keyof MiInstance): Promise<void> {
const { added, removed } = diffArrays(before, after);
if (removed.length > 0) {
await this.updateInstancesByHost(tem, field, false, removed);
}
if (added.length > 0) {
await this.updateInstancesByHost(tem, field, true, added);
}
}
private async updateInstancesByHost(tem: EntityManager, field: keyof MiInstance, value: boolean, hosts: string[]): Promise<void> {
// Use non-array queries when possible, as they are indexed and can be much faster.
if (hosts.length === 1) {
const pattern = genHostPattern(hosts[0]);
await tem
.createQueryBuilder(MiInstance, 'instance')
.update()
.set({ [field]: value })
.where('(lower(reverse("host")) || \'.\') LIKE :pattern', { pattern })
.execute();
} else if (hosts.length > 1) {
const patterns = hosts.map(host => genHostPattern(host));
await tem
.createQueryBuilder(MiInstance, 'instance')
.update()
.set({ [field]: value })
.where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', { patterns })
.execute();
}
}
}
function genHostPattern(host: string): string {
return host.toLowerCase().split('').reverse().join('') + '.%';
}

61
packages/backend/src/core/QueryService.ts
View file

@ -243,47 +243,44 @@ export class QueryService {
q.andWhere(new Brackets(qb => {
qb
.where(new Brackets(qb => {
qb.where('note.renoteId IS NOT NULL');
qb.andWhere('note.text IS NULL');
qb.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`);
}))
.orWhere('note.renoteId IS NULL')
.orWhere('note.text IS NOT NULL');
.orWhere('note.text IS NOT NULL')
.orWhere('note.cw IS NOT NULL')
.orWhere('note.replyId IS NOT NULL')
.orWhere('note.hasPoll = false')
.orWhere('note.fileIds != \'{}\'')
.orWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`);
}));
q.setParameters(mutingQuery.getParameters());
}
@bindThis
public generateBlockedHostQueryForNote(q: SelectQueryBuilder<any>, excludeAuthor?: boolean): void {
let nonBlockedHostQuery: (part: string) => string;
if (this.meta.blockedHosts.length === 0) {
nonBlockedHostQuery = () => '1=1';
} else {
nonBlockedHostQuery = (match: string) => `('.' || ${match}) NOT ILIKE ALL(select '%.' || x from (select unnest("blockedHosts") as x from "meta") t)`;
public generateBlockedHostQueryForNote(q: SelectQueryBuilder<any>, excludeAuthor?: boolean, allowSilenced = true): void {
function checkFor(key: 'user' | 'replyUser' | 'renoteUser') {
q.leftJoin(`note.${key}Instance`, `${key}Instance`);
q.andWhere(new Brackets(qb => {
qb.orWhere(`note.${key}Id IS NULL`) // no corresponding user
.orWhere(`note.${key}Host IS NULL`); // local
if (allowSilenced) {
qb.orWhere(`${key}Instance.isBlocked = false`); // not blocked
} else {
qb.orWhere(new Brackets(qbb => qbb
.andWhere(`${key}Instance.isBlocked = false`) // not blocked
.andWhere(`${key}Instance.isSilenced = false`))); // not silenced
}
if (excludeAuthor) {
qb.orWhere(`note.userId = note.${key}Id`); // author
}
}));
}
if (excludeAuthor) {
const instanceSuspension = (user: string) => new Brackets(qb => qb
.where(`note.${user}Id IS NULL`) // no corresponding user
.orWhere(`note.userId = note.${user}Id`)
.orWhere(`note.${user}Host IS NULL`) // local
.orWhere(nonBlockedHostQuery(`note.${user}Host`)));
q
.andWhere(instanceSuspension('replyUser'))
.andWhere(instanceSuspension('renoteUser'));
} else {
const instanceSuspension = (user: string) => new Brackets(qb => qb
.where(`note.${user}Id IS NULL`) // no corresponding user
.orWhere(`note.${user}Host IS NULL`) // local
.orWhere(nonBlockedHostQuery(`note.${user}Host`)));
q
.andWhere(instanceSuspension('user'))
.andWhere(instanceSuspension('replyUser'))
.andWhere(instanceSuspension('renoteUser'));
if (!excludeAuthor) {
checkFor('user');
}
checkFor('replyUser');
checkFor('renoteUser');
}
}

44
packages/backend/src/core/ReactionService.ts
View file

@ -10,7 +10,7 @@ import { IdentifiableError } from '@/misc/identifiable-error.js';
import type { MiRemoteUser, MiUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import { IdService } from '@/core/IdService.js';
import type { MiNoteReaction } from '@/models/NoteReaction.js';
import { MiNoteReaction } from '@/models/NoteReaction.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { NotificationService } from '@/core/NotificationService.js';
@ -31,6 +31,7 @@ import { isQuote, isRenote } from '@/misc/is-renote.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js';
import { CacheService } from '@/core/CacheService.js';
import type { DataSource } from 'typeorm';
const FALLBACK = '\u2764';
@ -89,6 +90,9 @@ export class ReactionService {
@Inject(DI.emojisRepository)
private emojisRepository: EmojisRepository,
@Inject(DI.db)
private readonly db: DataSource,
private utilityService: UtilityService,
private customEmojiService: CustomEmojiService,
private roleService: RoleService,
@ -176,26 +180,28 @@ export class ReactionService {
reaction,
};
try {
await this.noteReactionsRepository.insert(record);
} catch (e) {
if (isDuplicateKeyValueError(e)) {
const exists = await this.noteReactionsRepository.findOneByOrFail({
noteId: note.id,
userId: user.id,
});
const result = await this.db.transaction(async tem => {
await tem.createQueryBuilder(MiNoteReaction, 'noteReaction')
.insert()
.values(record)
.orIgnore()
.execute();
if (exists.reaction !== reaction) {
// 別のリアクションがすでにされていたら置き換える
await this.delete(user, note);
await this.noteReactionsRepository.insert(record);
} else {
// 同じリアクションがすでにされていたらエラー
throw new IdentifiableError('51c42bb4-931a-456b-bff7-e5a8a70dd298');
}
} else {
throw e;
return await tem.createQueryBuilder(MiNoteReaction, 'noteReaction')
.select()
.where({ noteId: note.id, userId: user.id })
.getOneOrFail();
});
if (result.id !== record.id) {
// Conflict with the same ID => nothing to do.
if (result.reaction === record.reaction) {
return;
}
// 別のリアクションがすでにされていたら置き換える
await this.delete(user, note);
await this.noteReactionsRepository.insert(record);
}
// Increment reactions count

2
packages/backend/src/core/ReversiService.ts
View file

@ -587,6 +587,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
lastActiveDate: parsed.user1.lastActiveDate != null ? new Date(parsed.user1.lastActiveDate) : null,
lastFetchedAt: parsed.user1.lastFetchedAt != null ? new Date(parsed.user1.lastFetchedAt) : null,
movedAt: parsed.user1.movedAt != null ? new Date(parsed.user1.movedAt) : null,
instance: null,
} : null,
user2: parsed.user2 != null ? {
...parsed.user2,
@ -597,6 +598,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
lastActiveDate: parsed.user2.lastActiveDate != null ? new Date(parsed.user2.lastActiveDate) : null,
lastFetchedAt: parsed.user2.lastFetchedAt != null ? new Date(parsed.user2.lastFetchedAt) : null,
movedAt: parsed.user2.movedAt != null ? new Date(parsed.user2.movedAt) : null,
instance: null,
} : null,
};
} else {

4
packages/backend/src/core/RoleService.ts
View file

@ -86,10 +86,10 @@ export const DEFAULT_POLICIES: RolePolicies = {
canManageCustomEmojis: false,
canManageAvatarDecorations: false,
canSearchNotes: false,
canUseTranslator: true,
canUseTranslator: false,
canHideAds: false,
driveCapacityMb: 100,
maxFileSizeMb: 10,
maxFileSizeMb: 25,
alwaysMarkNsfw: false,
canUpdateBioMedia: true,
pinLimit: 5,

39
packages/backend/src/core/UtilityService.ts
View file

@ -49,22 +49,49 @@ export class UtilityService {
return regexp.test(email);
}
public isBlockedHost(host: string | null): boolean;
public isBlockedHost(blockedHosts: string[], host: string | null): boolean;
@bindThis
public isBlockedHost(blockedHosts: string[], host: string | null): boolean {
public isBlockedHost(blockedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const blockedHosts = Array.isArray(blockedHostsOrHost) ? blockedHostsOrHost : this.meta.blockedHosts;
host = Array.isArray(blockedHostsOrHost) ? host : blockedHostsOrHost;
if (host == null) return false;
return blockedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
public isSilencedHost(host: string | null): boolean;
public isSilencedHost(silencedHosts: string[], host: string | null): boolean;
@bindThis
public isSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean {
if (!silencedHosts || host == null) return false;
public isSilencedHost(silencedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const silencedHosts = Array.isArray(silencedHostsOrHost) ? silencedHostsOrHost : this.meta.silencedHosts;
host = Array.isArray(silencedHostsOrHost) ? host : silencedHostsOrHost;
if (host == null) return false;
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
public isMediaSilencedHost(host: string | null): boolean;
public isMediaSilencedHost(silencedHosts: string[], host: string | null): boolean;
@bindThis
public isMediaSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean {
if (!silencedHosts || host == null) return false;
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
public isMediaSilencedHost(mediaSilencedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const mediaSilencedHosts = Array.isArray(mediaSilencedHostsOrHost) ? mediaSilencedHostsOrHost : this.meta.mediaSilencedHosts;
host = Array.isArray(mediaSilencedHostsOrHost) ? host : mediaSilencedHostsOrHost;
if (host == null) return false;
return mediaSilencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isAllowListedHost(host: string | null): boolean {
if (host == null) return false;
return this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isBubbledHost(host: string | null): boolean {
if (host == null) return false;
return this.meta.bubbleInstances.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis

64
packages/backend/src/core/VideoProcessingService.ts
View file

@ -3,24 +3,41 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import fs from 'node:fs/promises';
import { Inject, Injectable } from '@nestjs/common';
import FFmpeg from 'fluent-ffmpeg';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { ImageProcessingService } from '@/core/ImageProcessingService.js';
import type { IImage } from '@/core/ImageProcessingService.js';
import { createTempDir } from '@/misc/create-temp.js';
import { createTemp, createTempDir } from '@/misc/create-temp.js';
import { bindThis } from '@/decorators.js';
import { appendQuery, query } from '@/misc/prelude/url.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
// faststart is only supported for MP4, M4A, M4W and MOV files (the MOV family).
// WebM (and Matroska) files always support faststart-like behavior.
const supportedMimeTypes = new Map([
['video/mp4', 'mp4'],
['video/m4a', 'mp4'],
['video/m4v', 'mp4'],
['video/quicktime', 'mov'],
]);
@Injectable()
export class VideoProcessingService {
private readonly logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
private imageProcessingService: ImageProcessingService,
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('video-processing');
}
@bindThis
@ -60,5 +77,50 @@ export class VideoProcessingService {
}),
);
}
/**
* Optimize video for web playback by adding faststart flag.
* This allows the video to start playing before it is fully downloaded.
* The original file is modified in-place.
* @param source Path to the video file
* @param mimeType The MIME type of the video
* @returns Promise that resolves when optimization is complete
*/
@bindThis
public async webOptimizeVideo(source: string, mimeType: string): Promise<void> {
const outputFormat = supportedMimeTypes.get(mimeType);
if (!outputFormat) {
this.logger.debug(`Skipping web optimization for unsupported MIME type: ${mimeType}`);
return;
}
const [tempPath, cleanup] = await createTemp();
try {
await new Promise<void>((resolve, reject) => {
FFmpeg(source)
.format(outputFormat) // Specify output format
.addOutputOptions('-c copy') // Copy streams without re-encoding
.addOutputOptions('-movflags +faststart')
.on('error', reject)
.on('end', async () => {
try {
// Replace original file with optimized version
await fs.copyFile(tempPath, source);
this.logger.info(`Web-optimized video: ${source}`);
resolve();
} catch (copyError) {
reject(copyError);
}
})
.save(tempPath);
});
} catch (error) {
this.logger.warn(`Failed to web-optimize video: ${source}`, { error });
throw error;
} finally {
cleanup();
}
}
}

5
packages/backend/src/core/WebhookTestService.ts
View file

@ -63,6 +63,7 @@ function generateDummyUser(override?: Partial<MiUser>): MiUser {
emojis: [],
score: 0,
host: null,
instance: null,
inbox: null,
sharedInbox: null,
featured: null,
@ -115,10 +116,13 @@ function generateDummyNote(override?: Partial<MiNote>): MiNote {
channelId: null,
channel: null,
userHost: null,
userInstance: null,
replyUserId: null,
replyUserHost: null,
replyUserInstance: null,
renoteUserId: null,
renoteUserHost: null,
renoteUserInstance: null,
updatedAt: null,
processErrors: [],
...override,
@ -450,6 +454,7 @@ export class WebhookTestService {
isAdmin: false,
isModerator: false,
isSystem: false,
instance: undefined,
...override,
};
}

37
packages/backend/src/core/activitypub/ApInboxService.ts
View file

@ -36,7 +36,7 @@ import InstanceChart from '@/core/chart/charts/instance.js';
import FederationChart from '@/core/chart/charts/federation.js';
import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js';
import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { ApNoteService } from './models/ApNoteService.js';
import { ApLoggerService } from './ApLoggerService.js';
import { ApDbResolverService } from './ApDbResolverService.js';
@ -106,22 +106,25 @@ export class ApInboxService {
let result = undefined as string | void;
if (isCollectionOrOrderedCollection(activity)) {
const results = [] as [string, string | void][];
// eslint-disable-next-line no-param-reassign
resolver ??= this.apResolverService.createResolver();
const items = toArray(isCollection(activity) ? activity.items : activity.orderedItems);
if (items.length >= resolver.getRecursionLimit()) {
throw new Error(`skipping activity: collection would surpass recursion limit: ${this.utilityService.extractDbHost(actor.uri)}`);
}
for (const item of items) {
const act = await resolver.resolve(item);
if (act.id == null || this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) {
this.logger.debug('skipping activity: activity id is null or mismatching');
continue;
const items = await resolver.resolveCollectionItems(activity);
for (let i = 0; i < items.length; i++) {
const act = items[i];
if (act.id != null) {
if (this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) {
this.logger.warn('skipping activity: activity id mismatch');
continue;
}
} else {
// Activity ID should only be string or undefined.
act.id = undefined;
}
try {
results.push([getApId(item), await this.performOneActivity(actor, act, resolver)]);
const id = getNullableApId(act) ?? `${getNullableApId(activity)}#${i}`;
const result = await this.performOneActivity(actor, act, resolver);
results.push([id, result]);
} catch (err) {
if (err instanceof Error || typeof err === 'string') {
this.logger.error(err);
@ -217,6 +220,10 @@ export class ApInboxService {
const note = await this.apNoteService.resolveNote(object, { resolver });
if (!note) return `skip: target note not found ${targetUri}`;
if (note.userHost == null && note.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot react to local-only note');
}
await this.apNoteService.extractEmojis(activity.tag ?? [], actor.host).catch(() => null);
try {
@ -371,6 +378,10 @@ export class ApInboxService {
return 'skip: invalid actor for this activity';
}
if (renote.userHost == null && renote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot renote a local-only note');
}
this.logger.info(`Creating the (Re)Note: ${uri}`);
const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc, resolver);

17
packages/backend/src/core/activitypub/ApRequestService.ts
View file

@ -155,6 +155,8 @@ export class ApRequestService {
@bindThis
public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, digest?: string): Promise<void> {
this.apUtilityService.assertApUrl(url);
const body = typeof object === 'string' ? object : JSON.stringify(object);
const keypair = await this.userKeypairService.getUserKeypair(user.id);
@ -182,10 +184,13 @@ export class ApRequestService {
* Get AP object with http-signature
* @param user http-signature user
* @param url URL to fetch
* @param followAlternate
* @param allowAnonymous If a fetched object lacks an ID, then it will be auto-generated from the final URL. (default: false)
* @param followAlternate Whether to resolve HTML responses to their referenced canonical AP endpoint. (default: true)
*/
@bindThis
public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise<IObjectWithId> {
public async signedGet(url: string, user: { id: MiUser['id'] }, allowAnonymous = false, followAlternate?: boolean): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const _followAlternate = followAlternate ?? true;
const keypair = await this.userKeypairService.getUserKeypair(user.id);
@ -254,7 +259,7 @@ export class ApRequestService {
if (alternate) {
const href = alternate.getAttribute('href');
if (href && this.apUtilityService.haveSameAuthority(url, href)) {
return await this.signedGet(href, user, false);
return await this.signedGet(href, user, allowAnonymous, false);
}
}
} catch {
@ -271,7 +276,11 @@ export class ApRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity as IObjectWithId;
}

161
packages/backend/src/core/activitypub/ApResolverService.ts
View file

@ -5,6 +5,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm';
import promiseLimit from 'promise-limit';
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js';
import type { Config } from '@/config.js';
@ -19,11 +20,12 @@ import { ApLogService, calculateDurationSince, extractObjectContext } from '@/co
import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { getApId, getNullableApId, IObjectWithId, isCollectionOrOrderedCollection } from './type.js';
import { toArray } from '@/misc/prelude/array.js';
import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js';
import type { IObject, ICollection, IOrderedCollection, ApObject } from './type.js';
import type { IObject, ApObject, IAnonymousObject } from './type.js';
export class Resolver {
private history: Set<string>;
@ -63,11 +65,16 @@ export class Resolver {
return this.recursionLimit;
}
public async resolveCollection(value: string | IObjectWithId, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection & IObjectWithId>;
public async resolveCollection(value: string | IObject, allowAnonymous: boolean | undefined, sentFromUri: string): Promise<AnyCollection & IObjectWithId>;
public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection>;
@bindThis
public async resolveCollection(value: string | IObject): Promise<ICollection | IOrderedCollection> {
public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection> {
const collection = typeof value === 'string'
? await this.resolve(value)
: value;
? sentFromUri
? await this.secureResolve(value, sentFromUri, allowAnonymous)
: await this.resolve(value, allowAnonymous)
: value; // TODO try and remove this eventually, as it's a major security foot-gun
if (isCollectionOrOrderedCollection(collection)) {
return collection;
@ -76,20 +83,110 @@ export class Resolver {
}
}
public async resolveCollectionItems(collection: IAnonymousObject, limit?: number | null, allowAnonymousItems?: true, concurrency?: number): Promise<IAnonymousObject[]>;
public async resolveCollectionItems(collection: string | IObjectWithId, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise<IObjectWithId[]>;
public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise<IObject[]>;
/**
* Recursively resolves items from a collection.
* Stops when reaching the resolution limit or an optional item limit - whichever is lower.
* This method supports Collection, OrderedCollection, and individual pages of either type.
* Malformed collections (mixing Ordered and un-Ordered types) are also supported.
* @param collection Collection to resolve from - can be a URL or object of any supported collection type.
* @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit.
* @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID.
* @param concurrency Maximum number of items to resolve at once. (default: 4)
*/
@bindThis
public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency = 4): Promise<IObject[]> {
const resolvedItems: IObject[] = [];
// This is pulled up to avoid code duplication below
const iterate = async(items: ApObject, current: AnyCollection) => {
const sentFrom = current.id;
const itemArr = toArray(items);
const itemLimit = limit ?? Number.MAX_SAFE_INTEGER;
const allowAnonymous = allowAnonymousItems ?? false;
await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems);
};
let current: AnyCollection | null = await this.resolveCollection(collection);
do {
// Iterate all items in the current page
if (current.items) {
await iterate(current.items, current);
}
if (current.orderedItems) {
await iterate(current.orderedItems, current);
}
if (this.history.size >= this.recursionLimit) {
// Stop when we reach the fetch limit
current = null;
} else if (limit != null && resolvedItems.length >= limit) {
// Stop when we reach the item limit
current = null;
} else if (isCollection(current) || isOrderedCollection(current)) {
// Continue to first page
current = current.first ? await this.resolveCollection(current.first, true, current.id) : null;
} else if (isCollectionPage(current) || isOrderedCollectionPage(current)) {
// Continue to next page
current = current.next ? await this.resolveCollection(current.next, true, current.id) : null;
} else {
// Stop in all other conditions
current = null;
}
} while (current != null);
return resolvedItems;
}
private async resolveItemArray(source: (string | IObject)[], sentFrom: undefined, itemLimit: number, concurrency: number, allowAnonymousItems: true, destination: IAnonymousObject[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObjectWithId[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise<void> {
const recursionLimit = this.recursionLimit - this.history.size;
const batchLimit = Math.min(source.length, recursionLimit, itemLimit);
const limiter = promiseLimit<IObject>(concurrency);
const batch = await Promise.all(source
.slice(0, batchLimit)
.map(item => limiter(async () => {
if (sentFrom) {
// Use secureResolve to avoid re-fetching items that were included inline.
return await this.secureResolve(item, sentFrom, allowAnonymousItems);
} else if (allowAnonymousItems) {
return await this.resolveAnonymous(item);
} else {
// ID is required if we have neither sentFrom not allowAnonymousItems
const id = getApId(item);
return await this.resolve(id);
}
})));
destination.push(...batch);
};
/**
* Securely resolves an AP object or URL that has been sent from another instance.
* An input object is trusted if and only if its ID matches the authority of sentFromUri.
* In all other cases, the object is re-fetched from remote by input string or object ID.
* @param input The input object or URL to resolve
* @param sentFromUri The URL where this object originated. This MUST be accurate - all security checks depend on this value!
* @param allowAnonymous If true, anonymous objects are allowed and will have their ID set to sentFromUri. If false (default) then anonymous objects will be rejected with an error.
*/
@bindThis
public async secureResolve(input: ApObject, sentFromUri: string): Promise<IObjectWithId> {
public async secureResolve(input: string | IObject | [string | IObject], sentFromUri: string, allowAnonymous?: boolean): Promise<IObjectWithId> {
// Unpack arrays to get the value element.
const value = fromTuple(input);
if (value == null) {
throw new IdentifiableError('20058164-9de1-4573-8715-425753a21c1d', 'Cannot resolve null input');
// If anonymous input is allowed, then any object is automatically valid if we set the ID.
// We can short-circuit here and avoid un-necessary checks.
if (allowAnonymous && typeof(value) === 'object' && value.id == null) {
value.id = sentFromUri;
return value as IObjectWithId;
}
// This will throw if the input has no ID, which is good because we can't verify an anonymous object anyway.
// This ensures the input has a string ID, protecting against type confusion and rejecting anonymous objects.
const id = getApId(value);
// Check if we can use the provided object as-is.
@ -100,28 +197,52 @@ export class Resolver {
}
// If the checks didn't pass, then we must fetch the object and use that.
return await this.resolve(id);
return await this.resolve(id, allowAnonymous);
}
public async resolve(value: string | [string]): Promise<IObjectWithId>;
public async resolve(value: string | IObject | [string | IObject]): Promise<IObject>;
/**
* Resolves an anonymous object.
* The returned value will not have any ID present.
* If one is provided in the response, it will be removed automatically.
*/
@bindThis
public async resolve(value: string | IObject | [string | IObject]): Promise<IObject> {
public async resolveAnonymous(value: string | IObject | [string | IObject]): Promise<IAnonymousObject> {
value = fromTuple(value);
const object = await this.resolve(value);
object.id = undefined;
return object as IAnonymousObject;
}
public async resolve(value: string | [string], allowAnonymous?: boolean): Promise<IObjectWithId>;
public async resolve(value: string | IObjectWithId | [string | IObjectWithId], allowAnonymous?: boolean): Promise<IObjectWithId>;
public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean): Promise<IObject>;
/**
* Resolves a URL or object to an AP object.
* Tuples are expanded to their first element before anything else, and non-string inputs are returned as-is.
* Otherwise, the string URL is fetched and validated to represent a valid ActivityPub object.
* @param value The input value to resolve
* @param allowAnonymous Determines what to do if a response object lacks an ID field. If false (default), then an exception is thrown. If true, then the ID is populated from the final response URL.
*/
@bindThis
public async resolve(value: string | IObject | [string | IObject], allowAnonymous = false): Promise<IObject> {
value = fromTuple(value);
// TODO try and remove this eventually, as it's a major security foot-gun
if (typeof value !== 'string') {
return value;
}
const host = this.utilityService.extractDbHost(value);
if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) {
return await this._resolveLogged(value, host);
return await this._resolveLogged(value, host, allowAnonymous);
} else {
return await this._resolve(value, host);
return await this._resolve(value, host, allowAnonymous);
}
}
private async _resolveLogged(requestUri: string, host: string): Promise<IObjectWithId> {
private async _resolveLogged(requestUri: string, host: string, allowAnonymous: boolean): Promise<IObjectWithId> {
const startTime = process.hrtime.bigint();
const log = await this.apLogService.createFetchLog({
@ -130,7 +251,7 @@ export class Resolver {
});
try {
const result = await this._resolve(requestUri, host, log);
const result = await this._resolve(requestUri, host, allowAnonymous, log);
log.accepted = true;
log.result = 'ok';
@ -150,7 +271,7 @@ export class Resolver {
}
}
private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise<IObjectWithId> {
private async _resolve(value: string, host: string, allowAnonymous: boolean, log?: SkApFetchLog): Promise<IObjectWithId> {
if (value.includes('#')) {
// URLs with fragment parts cannot be resolved correctly because
// the fragment part does not get transmitted over HTTP(S).
@ -181,8 +302,8 @@ export class Resolver {
}
const object = (this.user
? await this.apRequestService.signedGet(value, this.user)
: await this.httpRequestService.getActivityJson(value));
? await this.apRequestService.signedGet(value, this.user, allowAnonymous)
: await this.httpRequestService.getActivityJson(value, false, allowAnonymous));
if (log) {
const { object: objectOnly, context, contextHash } = extractObjectContext(object);

32
packages/backend/src/core/activitypub/ApUtilityService.ts
View file

@ -77,16 +77,42 @@ export class ApUtilityService {
return acceptableUrls[0]?.url ?? null;
}
/**
* Verifies that a provided URL is in a format acceptable for federation.
* @throws {IdentifiableError} If URL cannot be parsed
* @throws {IdentifiableError} If URL is not HTTPS
*/
public assertApUrl(url: string | URL): void {
// If string, parse and validate
if (typeof(url) === 'string') {
try {
url = new URL(url);
} catch {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: not a valid URL`);
}
}
// Must be HTTPS
if (!this.checkHttps(url)) {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: unsupported protocol ${url.protocol}`);
}
}
/**
* Checks if the URL contains HTTPS.
* Additionally, allows HTTP in non-production environments.
* Based on check-https.ts.
*/
private checkHttps(url: string): boolean {
private checkHttps(url: string | URL): boolean {
const isNonProd = this.envService.env.NODE_ENV !== 'production';
// noinspection HttpUrlsUsage
return url.startsWith('https://') || (url.startsWith('http://') && isNonProd);
try {
const proto = new URL(url).protocol;
return proto === 'https:' || (proto === 'http:' && isNonProd);
} catch {
// Invalid URLs don't "count" as HTTPS
return false;
}
}
}

12
packages/backend/src/core/activitypub/models/ApNoteService.ts
View file

@ -95,6 +95,7 @@ export class ApNoteService {
actor?: MiRemoteUser,
user?: MiRemoteUser,
): Error | null {
this.apUtilityService.assertApUrl(uri);
const expectHost = this.utilityService.extractDbHost(uri);
const apType = getApType(object);
@ -284,6 +285,13 @@ export class ApNoteService {
const quote = await this.getQuote(note, entryUri, resolver);
const processErrors = quote === null ? ['quoteUnavailable'] : null;
if (reply && reply.userHost == null && reply.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot reply to local-only note');
}
if (quote && quote.userHost == null && quote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note');
}
// vote
if (reply && reply.hasPoll) {
const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id });
@ -481,6 +489,10 @@ export class ApNoteService {
const quote = await this.getQuote(note, entryUri, resolver);
const processErrors = quote === null ? ['quoteUnavailable'] : null;
if (quote && quote.userHost == null && quote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note');
}
// vote
if (reply && reply.hasPoll) {
const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id });

48
packages/backend/src/core/activitypub/models/ApPersonService.ts
View file

@ -153,6 +153,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
*/
@bindThis
private validateActor(x: IObject, uri: string): IActor {
this.apUtilityService.assertApUrl(uri);
const expectHost = this.utilityService.punyHostPSLDomain(uri);
if (!isActor(x)) {
@ -167,6 +168,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`);
}
this.apUtilityService.assertApUrl(x.inbox);
const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox);
if (inboxHost !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`);
@ -175,6 +177,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined);
if (sharedInboxObject != null) {
const sharedInbox = getApId(sharedInboxObject);
this.apUtilityService.assertApUrl(sharedInbox);
if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`);
}
@ -185,6 +188,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
if (xCollection != null) {
const collectionUri = getApId(xCollection);
if (typeof collectionUri === 'string' && collectionUri.length > 0) {
this.apUtilityService.assertApUrl(collectionUri);
if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`);
}
@ -352,8 +356,8 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const [followingVisibility, followersVisibility] = await Promise.all(
[
this.isPublicCollection(person.following, resolver),
this.isPublicCollection(person.followers, resolver),
this.isPublicCollection(person.following, resolver, uri),
this.isPublicCollection(person.followers, resolver, uri),
].map((p): Promise<'public' | 'private'> => p
.then(isPublic => isPublic ? 'public' : 'private')
.catch(err => {
@ -389,10 +393,18 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
//#endregion
//#region resolve counts
const _resolver = resolver ?? this.apResolverService.createResolver();
const outboxcollection = await _resolver.resolveCollection(person.outbox).catch(() => { return null; });
const followerscollection = await _resolver.resolveCollection(person.followers!).catch(() => { return null; });
const followingcollection = await _resolver.resolveCollection(person.following!).catch(() => { return null; });
const outboxCollection = person.outbox
? await resolver.resolveCollection(person.outbox, true, uri).catch(() => { return null; })
: null;
const followersCollection = person.followers
? await resolver.resolveCollection(person.followers, true, uri).catch(() => { return null; })
: null;
const followingCollection = person.following
? await resolver.resolveCollection(person.following, true, uri).catch(() => { return null; })
: null;
// Register the instance first, to avoid FK errors
await this.federatedInstanceService.fetchOrRegister(host);
try {
// Start transaction
@ -419,9 +431,9 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
host,
inbox: person.inbox,
sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null,
notesCount: outboxcollection?.totalItems ?? 0,
followersCount: followerscollection?.totalItems ?? 0,
followingCount: followingcollection?.totalItems ?? 0,
notesCount: outboxCollection?.totalItems ?? 0,
followersCount: followersCollection?.totalItems ?? 0,
followingCount: followingCollection?.totalItems ?? 0,
followersUri: person.followers ? getApId(person.followers) : undefined,
featured: person.featured ? getApId(person.featured) : undefined,
uri: person.id,
@ -571,8 +583,8 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const [followingVisibility, followersVisibility] = await Promise.all(
[
this.isPublicCollection(person.following, resolver),
this.isPublicCollection(person.followers, resolver),
this.isPublicCollection(person.following, resolver, exist.uri),
this.isPublicCollection(person.followers, resolver, exist.uri),
].map((p): Promise<'public' | 'private' | undefined> => p
.then(isPublic => isPublic ? 'public' : 'private')
.catch(err => {
@ -797,13 +809,13 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const _resolver = resolver ?? this.apResolverService.createResolver();
// Resolve to (Ordered)Collection Object
const collection = await _resolver.resolveCollection(user.featured).catch(err => {
const collection = user.featured ? await _resolver.resolveCollection(user.featured, true, user.uri).catch(err => {
if (err instanceof AbortError || err instanceof StatusError) {
this.logger.warn(`Failed to update featured notes: ${err.name}: ${err.message}`);
} else {
this.logger.error('Failed to update featured notes:', err);
}
});
}) : null;
if (!collection) return;
if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`featured ${user.featured} is not Collection or OrderedCollection in ${user.uri}`);
@ -889,11 +901,13 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
}
@bindThis
private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver): Promise<boolean> {
private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver, sentFrom: string): Promise<boolean> {
if (collection) {
const resolved = await resolver.resolveCollection(collection);
if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) {
return true;
const resolved = await resolver.resolveCollection(collection, true, sentFrom).catch(() => null);
if (resolved) {
if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) {
return true;
}
}
}

60
packages/backend/src/core/activitypub/type.ts
View file

@ -43,6 +43,18 @@ export interface IObjectWithId extends IObject {
id: string;
}
export function isObjectWithId(object: IObject): object is IObjectWithId {
return typeof(object.id) === 'string';
}
export interface IAnonymousObject extends IObject {
id: undefined;
}
export function isAnonymousObject(object: IObject): object is IAnonymousObject {
return object.id === undefined;
}
/**
* Get array of ActivityStreams Objects id
*/
@ -125,48 +137,46 @@ export interface IActivity extends IObject {
};
}
export interface ICollection extends IObject {
export interface CollectionBase extends IObject {
totalItems?: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: ApObject;
orderedItems?: ApObject;
}
export interface ICollection extends CollectionBase {
type: 'Collection';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
items?: ApObject;
orderedItems?: undefined;
}
export interface IOrderedCollection extends IObject {
export interface IOrderedCollection extends CollectionBase {
type: 'OrderedCollection';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
items?: undefined;
orderedItems?: ApObject;
}
export interface ICollectionPage extends IObject {
export interface ICollectionPage extends CollectionBase {
type: 'CollectionPage';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: ApObject;
orderedItems?: undefined;
}
export interface IOrderedCollectionPage extends IObject {
export interface IOrderedCollectionPage extends CollectionBase {
type: 'OrderedCollectionPage';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: undefined;
orderedItems?: ApObject;
}
export type AnyCollection = ICollection | IOrderedCollection | ICollectionPage | IOrderedCollectionPage;
export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event'];
export const isPost = (object: IObject): object is IPost => {
@ -270,7 +280,7 @@ export const isCollectionPage = (object: IObject): object is ICollectionPage =>
export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage =>
getApType(object) === 'OrderedCollectionPage';
export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection =>
export const isCollectionOrOrderedCollection = (object: IObject): object is AnyCollection =>
isCollection(object) || isOrderedCollection(object) || isCollectionPage(object) || isOrderedCollectionPage(object);
export interface IApPropertyValue extends IObject {

23
packages/backend/src/core/chart/charts/federation.ts
View file

@ -44,10 +44,6 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
const suspendedInstancesQuery = this.instancesRepository.createQueryBuilder('instance')
.select('instance.host')
.where('instance.suspensionState != \'none\'');
const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f')
.select('f.followerHost')
.where('f.followerHost IS NOT NULL');
@ -64,22 +60,25 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followeeInstance', 'followeeInstance')
.andWhere('followeeInstance.suspensionState = \'none\'')
.andWhere('followeeInstance.isBlocked = false')
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followerHost)')
.where('following.followerHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followerHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followerHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followerInstance', 'followerInstance')
.andWhere('followerInstance.isBlocked = false')
.andWhere('followerInstance.suspensionState = \'none\'')
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followeeInstance', 'followeeInstance')
.andWhere('followeeInstance.isBlocked = false')
.andWhere('followeeInstance.suspensionState = \'none\'')
.andWhere(`following.followeeHost IN (${ pubsubSubQuery.getQuery() })`)
.setParameters(pubsubSubQuery.getParameters())
.getRawOne()
@ -87,7 +86,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ subInstancesQuery.getQuery() })`)
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere('instance.isBlocked = false')
.andWhere('instance.suspensionState = \'none\'')
.andWhere('instance.isNotResponding = false')
.getRawOne()
@ -95,7 +94,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ pubInstancesQuery.getQuery() })`)
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere('instance.isBlocked = false')
.andWhere('instance.suspensionState = \'none\'')
.andWhere('instance.isNotResponding = false')
.getRawOne()

7
packages/backend/src/core/entities/InstanceEntityService.ts
View file

@ -43,7 +43,7 @@ export class InstanceEntityService {
isNotResponding: instance.isNotResponding,
isSuspended: instance.suspensionState !== 'none',
suspensionState: instance.suspensionState,
isBlocked: this.utilityService.isBlockedHost(this.meta.blockedHosts, instance.host),
isBlocked: instance.isBlocked,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,
openRegistrations: instance.openRegistrations,
@ -51,8 +51,8 @@ export class InstanceEntityService {
description: instance.description,
maintainerName: instance.maintainerName,
maintainerEmail: instance.maintainerEmail,
isSilenced: this.utilityService.isSilencedHost(this.meta.silencedHosts, instance.host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(this.meta.mediaSilencedHosts, instance.host),
isSilenced: instance.isSilenced,
isMediaSilenced: instance.isMediaSilenced,
iconUrl: instance.iconUrl,
faviconUrl: instance.faviconUrl,
themeColor: instance.themeColor,
@ -62,6 +62,7 @@ export class InstanceEntityService {
rejectReports: instance.rejectReports,
rejectQuotes: instance.rejectQuotes,
moderationNote: iAmModerator ? instance.moderationNote : null,
isBubbled: this.utilityService.isBubbledHost(instance.host),
};
}

22
packages/backend/src/core/entities/NotificationEntityService.ts
View file

@ -23,6 +23,13 @@ import type { NoteEntityService } from './NoteEntityService.js';
const NOTE_REQUIRED_NOTIFICATION_TYPES = new Set(['note', 'mention', 'reply', 'renote', 'renote:grouped', 'quote', 'reaction', 'reaction:grouped', 'pollEnded', 'edited', 'scheduledNotePosted'] as (typeof groupedNotificationTypes[number])[]);
function undefOnMissing<T>(packPromise: Promise<T>): Promise<T | undefined> {
return packPromise.catch(err => {
if (err instanceof EntityNotFoundError) return undefined;
throw err;
});
}
@Injectable()
export class NotificationEntityService implements OnModuleInit {
private userEntityService: UserEntityService;
@ -75,9 +82,9 @@ export class NotificationEntityService implements OnModuleInit {
const noteIfNeed = needsNote ? (
hint?.packedNotes != null
? hint.packedNotes.get(notification.noteId)
: this.noteEntityService.pack(notification.noteId, { id: meId }, {
: undefOnMissing(this.noteEntityService.pack(notification.noteId, { id: meId }, {
detail: true,
})
}))
) : undefined;
// if the note has been deleted, don't show this notification
if (needsNote && !noteIfNeed) return null;
@ -86,7 +93,7 @@ export class NotificationEntityService implements OnModuleInit {
const userIfNeed = needsUser ? (
hint?.packedUsers != null
? hint.packedUsers.get(notification.notifierId)
: this.userEntityService.pack(notification.notifierId, { id: meId })
: undefOnMissing(this.userEntityService.pack(notification.notifierId, { id: meId }))
) : undefined;
// if the user has been deleted, don't show this notification
if (needsUser && !userIfNeed) return null;
@ -96,7 +103,7 @@ export class NotificationEntityService implements OnModuleInit {
const reactions = (await Promise.all(notification.reactions.map(async reaction => {
const user = hint?.packedUsers != null
? hint.packedUsers.get(reaction.userId)!
: await this.userEntityService.pack(reaction.userId, { id: meId });
: await undefOnMissing(this.userEntityService.pack(reaction.userId, { id: meId }));
return {
user,
reaction: reaction.reaction,
@ -121,7 +128,7 @@ export class NotificationEntityService implements OnModuleInit {
return packedUser;
}
return this.userEntityService.pack(userId, { id: meId });
return undefOnMissing(this.userEntityService.pack(userId, { id: meId }));
}))).filter(x => x != null);
// if all users have been deleted, don't show this notification
if (users.length === 0) {
@ -140,10 +147,7 @@ export class NotificationEntityService implements OnModuleInit {
const needsRole = notification.type === 'roleAssigned';
const role = needsRole
? await this.roleEntityService.pack(notification.roleId).catch(err => {
if (err instanceof EntityNotFoundError) return undefined;
throw err;
})
? await undefOnMissing(this.roleEntityService.pack(notification.roleId))
: undefined;
// if the role has been deleted, don't show this notification
if (needsRole && !role) {

2
packages/backend/src/core/entities/UserEntityService.ts
View file

@ -610,7 +610,7 @@ export class UserEntityService implements OnModuleInit {
requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true,
makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined,
makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined,
instance: user.host ? this.federatedInstanceService.federatedInstanceCache.fetch(user.host).then(instance => instance ? {
instance: user.host ? this.federatedInstanceService.fetch(user.host).then(instance => instance ? {
name: instance.name,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,