diff --git a/locales/index.d.ts b/locales/index.d.ts index 5691130b55..451f156b4b 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -13285,6 +13285,36 @@ export interface Locale extends ILocale { * Signup Reason */ "signupReason": string; + "clearCachedFilesOptions": { + /** + * Delete all cached remote files + */ + "title": string; + /** + * Only delete files older than: + */ + "olderThan": string; + /** + * now + */ + "now": string; + /** + * one week + */ + "oneWeek": string; + /** + * one month + */ + "oneMonth": string; + /** + * one year + */ + "oneYear": string; + /** + * Don't delete files used as avatars&c + */ + "keepFilesInUse": string; + }; } declare const locales: { [lang: string]: Locale; diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 361e636662..f4f069b64b 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -684,8 +684,11 @@ export class QueueService { } @bindThis - public createCleanRemoteFilesJob() { - return this.objectStorageQueue.add('cleanRemoteFiles', {}, { + public createCleanRemoteFilesJob(olderThanSeconds: number = 0, keepFilesInUse: boolean = false) { + return this.objectStorageQueue.add('cleanRemoteFiles', { + keepFilesInUse, + olderThanSeconds, + }, { removeOnComplete: { age: 3600 * 24 * 7, // keep up to 7 days count: 30, diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 2eddae95c8..354f351358 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -4,14 +4,17 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { IsNull, MoreThan, Not } from 'typeorm'; +import { IsNull, MoreThan, Not, Brackets } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { MiDriveFile, DriveFilesRepository } from '@/models/_.js'; +import { MiUser } from '@/models/_.js'; import type Logger from '@/logger.js'; import { DriveService } from '@/core/DriveService.js'; import { bindThis } from '@/decorators.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; +import type { CleanRemoteFilesJobData } from '../types.js'; +import { IdService } from '@/core/IdService.js'; @Injectable() export class CleanRemoteFilesProcessorService { @@ -23,35 +26,54 @@ export class CleanRemoteFilesProcessorService { private driveService: DriveService, private queueLoggerService: QueueLoggerService, + private idService: IdService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('clean-remote-files'); } @bindThis - public async process(job: Bull.Job>): Promise { + public async process(job: Bull.Job): Promise { this.logger.info('Deleting cached remote files...'); + const olderThanTimestamp = Date.now() - (job.data.olderThanSeconds ?? 0) * 1000; + const olderThanDate = new Date(olderThanTimestamp); + const keepFilesInUse = job.data.keepFilesInUse ?? false; let deletedCount = 0; let cursor: MiDriveFile['id'] | null = null; let errorCount = 0; - const total = await this.driveFilesRepository.countBy({ - userHost: Not(IsNull()), - isLink: false, - }); + const filesQuery = this.driveFilesRepository.createQueryBuilder('file') + .where('file.userHost IS NOT NULL') // remote files + .andWhere('file.isLink = FALSE') // cached + .andWhere('file.id <= :id', { id: this.idService.gen(olderThanTimestamp) }) // and old + .orderBy('file.id', 'ASC'); + + if (keepFilesInUse) { + filesQuery + // are they used as avatar&&c? + .leftJoinAndSelect( + MiUser, 'fileuser', + 'fileuser."avatarId"="file"."id" OR fileuser."bannerId"="file"."id" OR fileuser."backgroundId"="file"."id"' + ) + .andWhere( + new Brackets((qb) => { + qb.where('fileuser.id IS NULL') // not used + .orWhere( // or attached to a user + new Brackets((qb) => { + qb.where('fileuser.lastFetchedAt IS NOT NULL') // weird? maybe this only applies to local users + .andWhere('fileuser.lastFetchedAt < :old', { old: olderThanDate }); // old user + }) + ); + }) + ); + } + + const total = await filesQuery.clone().getCount(); while (true) { - const files = await this.driveFilesRepository.find({ - where: { - userHost: Not(IsNull()), - isLink: false, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 256, - order: { - id: 1, - }, - }); + const thisBatchQuery = filesQuery.clone(); + if (cursor) thisBatchQuery.andWhere('file.id > :cursor', { cursor }); + const files = await thisBatchQuery.take(256).getMany(); if (files.length === 0) { job.updateProgress(100); diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 1bd9f7a0ab..79ab68ab1d 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -40,6 +40,11 @@ export type RelationshipJobData = { withReplies?: boolean; }; +export type CleanRemoteFilesJobData = { + keepFilesInUse: boolean; + olderThanSeconds: number; +}; + export type DbJobData = DbJobMap[T]; export type DbJobMap = { diff --git a/packages/backend/src/server/api/endpoints/admin/drive/clean-remote-files.ts b/packages/backend/src/server/api/endpoints/admin/drive/clean-remote-files.ts index 9a7b3d5d62..27bbe8f29e 100644 --- a/packages/backend/src/server/api/endpoints/admin/drive/clean-remote-files.ts +++ b/packages/backend/src/server/api/endpoints/admin/drive/clean-remote-files.ts @@ -18,7 +18,10 @@ export const meta = { export const paramDef = { type: 'object', - properties: {}, + properties: { + olderThanSeconds: { type: 'number' }, + keepFilesInUse: { type: 'boolean' }, + }, required: [], } as const; @@ -30,7 +33,10 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { await this.moderationLogService.log(me, 'clearRemoteFiles', {}); - await this.queueService.createCleanRemoteFilesJob(); + await this.queueService.createCleanRemoteFilesJob( + ps.olderThanSeconds ?? 0, + ps.keepFilesInUse ?? false, + ); }); } } diff --git a/packages/frontend/src/pages/admin/files.vue b/packages/frontend/src/pages/admin/files.vue index 87595a820b..01745fcee2 100644 --- a/packages/frontend/src/pages/admin/files.vue +++ b/packages/frontend/src/pages/admin/files.vue @@ -58,14 +58,40 @@ const pagination = { })), }; -function clear() { - os.confirm({ - type: 'warning', - text: i18n.ts.clearCachedFilesConfirm, - }).then(({ canceled }) => { - if (canceled) return; +async function clear() { + const { canceled, result } = await os.form(i18n.ts.clearCachedFilesOptions.title, { + olderThanEnum: { + label: i18n.ts.clearCachedFilesOptions.olderThan, + type: 'enum', + default: 'now', + required: true, + enum: [ + { label: i18n.ts.clearCachedFilesOptions.now, value: 'now' }, + { label: i18n.ts.clearCachedFilesOptions.oneWeek, value: 'oneWeek' }, + { label: i18n.ts.clearCachedFilesOptions.oneMonth, value: 'oneMonth' }, + { label: i18n.ts.clearCachedFilesOptions.oneYear, value: 'oneYear' }, + ], + }, + keepFilesInUse: { + label: i18n.ts.clearCachedFilesOptions.keepFilesInUse, + description: i18n.ts.clearCachedFilesOptions.keepFilesInUseDescription, + type: 'boolean', + default: false, + }, + }); - os.apiWithDialog('admin/drive/clean-remote-files', {}); + if (canceled) return; + + const timesMap = { + now: 0, + oneWeek: 7 * 86400, + oneMonth: 30 * 86400, + oneYear: 365 * 86400, + }; + + await os.apiWithDialog('admin/drive/clean-remote-files', { + olderThanSeconds: timesMap[result.olderThanEnum] ?? 0, + keepFilesInUse: result.keepFilesInUse, }); } diff --git a/sharkey-locales/en-US.yml b/sharkey-locales/en-US.yml index f0aaf5e227..23a3afce82 100644 --- a/sharkey-locales/en-US.yml +++ b/sharkey-locales/en-US.yml @@ -636,3 +636,13 @@ rawInfoDescription: "Extended user data in its raw form. These fields are privat rawApDescription: "ActivityPub user data in its raw form. These fields are public and accessible to other instances." signupReason: "Signup Reason" + +clearCachedFilesOptions: + title: "Delete all cached remote files" + olderThan: "Only delete files older than:" + now: "now" + oneWeek: "one week" + oneMonth: "one month" + oneYear: "one year" + keepFilesInUse: "Don't delete files used as avatars&c" + keepFilesInUseDescription: "this option requires more complicated database queries, you may need to increase the value of db.extra.statement_timeout in the configuration file"