From c68b8d6e7ced1a6202b29faee7dfeddae114683e Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 24 Jun 2025 15:34:13 +0100 Subject: [PATCH 1/5] smarter "clean remote files" this should (optionally) skip in-use files, and files that have been seen recently --- packages/backend/src/core/QueueService.ts | 7 ++- .../CleanRemoteFilesProcessorService.ts | 56 +++++++++++++------ packages/backend/src/queue/types.ts | 5 ++ 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 361e636662..f4f069b64b 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -684,8 +684,11 @@ export class QueueService { } @bindThis - public createCleanRemoteFilesJob() { - return this.objectStorageQueue.add('cleanRemoteFiles', {}, { + public createCleanRemoteFilesJob(olderThanSeconds: number = 0, keepFilesInUse: boolean = false) { + return this.objectStorageQueue.add('cleanRemoteFiles', { + keepFilesInUse, + olderThanSeconds, + }, { removeOnComplete: { age: 3600 * 24 * 7, // keep up to 7 days count: 30, diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 2eddae95c8..4bd3938f0b 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -4,14 +4,17 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { IsNull, MoreThan, Not } from 'typeorm'; +import { IsNull, MoreThan, Not, Brackets } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { MiDriveFile, DriveFilesRepository } from '@/models/_.js'; +import { MiUser } from '@/models/_.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; +import type { CleanRemoteFilesJobData } from '../types.js'; +import { IdService } from '@/core/IdService.js'; @Injectable() export class CleanRemoteFilesProcessorService { @@ -23,35 +26,54 @@ export class CleanRemoteFilesProcessorService { private driveService: DriveService, private queueLoggerService: QueueLoggerService, + private idService: IdService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('clean-remote-files'); } @bindThis - public async process(job: Bull.Job>): Promise { + public async process(job: Bull.Job): Promise { this.logger.info('Deleting cached remote files...'); + const olderThanTimestamp = Date.now() - job.data.olderThanSeconds * 1000; + const olderThanDate = new Date(olderThanTimestamp); + const keepFilesInUse = job.data.keepFilesInUse; let deletedCount = 0; let cursor: MiDriveFile['id'] | null = null; let errorCount = 0; - const total = await this.driveFilesRepository.countBy({ - userHost: Not(IsNull()), - isLink: false, - }); + const filesQuery = this.driveFilesRepository.createQueryBuilder('file') + .where('file.userHost IS NOT NULL') // remote files + .andWhere('file.isLink = FALSE') // cached + .andWhere('file.id <= :id', { id: this.idService.gen(olderThanTimestamp) }) // and old + .orderBy('file.id', 'ASC'); + + if (keepFilesInUse) { + filesQuery + // are they used as avatar&&c? + .leftJoinAndSelect( + MiUser, 'fileuser', + 'fileuser."avatarId"="file"."id" OR fileuser."bannerId"="file"."id" OR fileuser."backgroundId"="file"."id"' + ) + .andWhere( + new Brackets((qb) => { + qb.where('fileuser.id IS NULL') // not used + .orWhere( // or attached to a user + new Brackets((qb) => { + qb.where('fileuser.lastFetchedAt IS NOT NULL') // weird? maybe this only applies to local users + .andWhere('fileuser.lastFetchedAt < :old', { old: olderThanDate }); // old user + }) + ); + }) + ); + } + + const total = await filesQuery.clone().getCount(); while (true) { - const files = await this.driveFilesRepository.find({ - where: { - userHost: Not(IsNull()), - isLink: false, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 256, - order: { - id: 1, - }, - }); + const thisBatchQuery = filesQuery.clone(); + if (cursor) thisBatchQuery.andWhere('file.id > :cursor', { cursor }); + const files = await thisBatchQuery.take(256).getMany(); if (files.length === 0) { job.updateProgress(100); diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 1bd9f7a0ab..79ab68ab1d 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -40,6 +40,11 @@ export type RelationshipJobData = { withReplies?: boolean; }; +export type CleanRemoteFilesJobData = { + keepFilesInUse: boolean; + olderThanSeconds: number; +}; + export type DbJobData = DbJobMap[T]; export type DbJobMap = { From 50ca5e868851f8e0f97b773971c8d4c8ab68d975 Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 24 Jun 2025 15:37:13 +0100 Subject: [PATCH 2/5] pass arguments to CleanRemoteFiles job --- .../api/endpoints/admin/drive/clean-remote-files.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/admin/drive/clean-remote-files.ts b/packages/backend/src/server/api/endpoints/admin/drive/clean-remote-files.ts index 9a7b3d5d62..27bbe8f29e 100644 --- a/packages/backend/src/server/api/endpoints/admin/drive/clean-remote-files.ts +++ b/packages/backend/src/server/api/endpoints/admin/drive/clean-remote-files.ts @@ -18,7 +18,10 @@ export const meta = { export const paramDef = { type: 'object', - properties: {}, + properties: { + olderThanSeconds: { type: 'number' }, + keepFilesInUse: { type: 'boolean' }, + }, required: [], } as const; @@ -30,7 +33,10 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { await this.moderationLogService.log(me, 'clearRemoteFiles', {}); - await this.queueService.createCleanRemoteFilesJob(); + await this.queueService.createCleanRemoteFilesJob( + ps.olderThanSeconds ?? 0, + ps.keepFilesInUse ?? false, + ); }); } } From d847dd00c551b635daf25df7a29d0727055757a1 Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 24 Jun 2025 16:16:14 +0100 Subject: [PATCH 3/5] frontend dialog for "clear cached files" --- locales/index.d.ts | 30 ++++++++++++++++ packages/frontend/src/pages/admin/files.vue | 39 +++++++++++++++++---- sharkey-locales/en-US.yml | 9 +++++ 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/locales/index.d.ts b/locales/index.d.ts index da0f7a963f..51043e51dc 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -13265,6 +13265,36 @@ export interface Locale extends ILocale { * Signup Reason */ "signupReason": string; + "clearCachedFilesOptions": { + /** + * Delete all cached remote files + */ + "title": string; + /** + * Only delete files older than: + */ + "olderThan": string; + /** + * now + */ + "now": string; + /** + * one week + */ + "oneWeek": string; + /** + * one month + */ + "oneMonth": string; + /** + * one year + */ + "oneYear": string; + /** + * Don't delete files used as avatars&c + */ + "keepFilesInUse": string; + }; } declare const locales: { [lang: string]: Locale; diff --git a/packages/frontend/src/pages/admin/files.vue b/packages/frontend/src/pages/admin/files.vue index 87595a820b..fb40bbeeed 100644 --- a/packages/frontend/src/pages/admin/files.vue +++ b/packages/frontend/src/pages/admin/files.vue @@ -58,14 +58,39 @@ const pagination = { })), }; -function clear() { - os.confirm({ - type: 'warning', - text: i18n.ts.clearCachedFilesConfirm, - }).then(({ canceled }) => { - if (canceled) return; +async function clear() { + const { canceled, result } = await os.form(i18n.ts.clearCachedFilesOptions.title, { + olderThanEnum: { + label: i18n.ts.clearCachedFilesOptions.olderThan, + type: 'enum', + default: 'now', + required: true, + enum: [ + { label: i18n.ts.clearCachedFilesOptions.now, value: 'now' }, + { label: i18n.ts.clearCachedFilesOptions.oneWeek, value: 'oneWeek' }, + { label: i18n.ts.clearCachedFilesOptions.oneMonth, value: 'oneMonth' }, + { label: i18n.ts.clearCachedFilesOptions.oneYear, value: 'oneYear' }, + ], + }, + keepFilesInUse: { + label: i18n.ts.clearCachedFilesOptions.keepFilesInUse, + type: 'boolean', + default: true, + }, + }); - os.apiWithDialog('admin/drive/clean-remote-files', {}); + if (canceled) return; + + const timesMap = { + now: 0, + oneWeek: 7 * 86400, + oneMonth: 30 * 86400, + oneYear: 365 * 86400, + }; + + await os.apiWithDialog('admin/drive/clean-remote-files', { + olderThanSeconds: timesMap[result.olderThanEnum] ?? 0, + keepFilesInUse: result.keepFilesInUse, }); } diff --git a/sharkey-locales/en-US.yml b/sharkey-locales/en-US.yml index c79302ff7a..ab6712225f 100644 --- a/sharkey-locales/en-US.yml +++ b/sharkey-locales/en-US.yml @@ -632,3 +632,12 @@ rawInfoDescription: "Extended user data in its raw form. These fields are privat rawApDescription: "ActivityPub user data in its raw form. These fields are public and accessible to other instances." signupReason: "Signup Reason" + +clearCachedFilesOptions: + title: "Delete all cached remote files" + olderThan: "Only delete files older than:" + now: "now" + oneWeek: "one week" + oneMonth: "one month" + oneYear: "one year" + keepFilesInUse: "Don't delete files used as avatars&c" From bddb6afa5d3ad22f30ce578c3b4cd6ced219b6ee Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 24 Jun 2025 17:16:30 +0100 Subject: [PATCH 4/5] handle jobs created without the extra arguments just in case there's any in the queue when people upgrade --- .../src/queue/processors/CleanRemoteFilesProcessorService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 4bd3938f0b..354f351358 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -35,9 +35,9 @@ export class CleanRemoteFilesProcessorService { public async process(job: Bull.Job): Promise { this.logger.info('Deleting cached remote files...'); - const olderThanTimestamp = Date.now() - job.data.olderThanSeconds * 1000; + const olderThanTimestamp = Date.now() - (job.data.olderThanSeconds ?? 0) * 1000; const olderThanDate = new Date(olderThanTimestamp); - const keepFilesInUse = job.data.keepFilesInUse; + const keepFilesInUse = job.data.keepFilesInUse ?? false; let deletedCount = 0; let cursor: MiDriveFile['id'] | null = null; let errorCount = 0; From a450f801cf56de4c7e75b32da377fa663bdc7487 Mon Sep 17 00:00:00 2001 From: dakkar Date: Fri, 27 Jun 2025 11:16:56 +0100 Subject: [PATCH 5/5] warn about extra db load --- packages/frontend/src/pages/admin/files.vue | 3 ++- sharkey-locales/en-US.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/frontend/src/pages/admin/files.vue b/packages/frontend/src/pages/admin/files.vue index fb40bbeeed..01745fcee2 100644 --- a/packages/frontend/src/pages/admin/files.vue +++ b/packages/frontend/src/pages/admin/files.vue @@ -74,8 +74,9 @@ async function clear() { }, keepFilesInUse: { label: i18n.ts.clearCachedFilesOptions.keepFilesInUse, + description: i18n.ts.clearCachedFilesOptions.keepFilesInUseDescription, type: 'boolean', - default: true, + default: false, }, }); diff --git a/sharkey-locales/en-US.yml b/sharkey-locales/en-US.yml index ab6712225f..091bbf8be5 100644 --- a/sharkey-locales/en-US.yml +++ b/sharkey-locales/en-US.yml @@ -641,3 +641,4 @@ clearCachedFilesOptions: oneMonth: "one month" oneYear: "one year" keepFilesInUse: "Don't delete files used as avatars&c" + keepFilesInUseDescription: "this option requires more complicated database queries, you may need to increase the value of db.extra.statement_timeout in the configuration file"