merge: smarter "clean remote files" - fixes #801 (!1143)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/1143

Closes #801

Approved-by: Hazelnoot <acomputerdog@gmail.com>
Approved-by: Marie <github@yuugi.dev>

locales/index.d.ts

@@ -13285,6 +13285,36 @@ export interface Locale extends ILocale {
      * Signup Reason
      */
     "signupReason": string;
+    "clearCachedFilesOptions": {
+        /**
+         * Delete all cached remote files
+         */
+        "title": string;
+        /**
+         * Only delete files older than:
+         */
+        "olderThan": string;
+        /**
+         * now
+         */
+        "now": string;
+        /**
+         * one week
+         */
+        "oneWeek": string;
+        /**
+         * one month
+         */
+        "oneMonth": string;
+        /**
+         * one year
+         */
+        "oneYear": string;
+        /**
+         * Don't delete files used as avatars&c
+         */
+        "keepFilesInUse": string;
+    };
 }
 declare const locales: {
     [lang: string]: Locale;

packages/backend/src/core/QueueService.ts

@@ -684,8 +684,11 @@ export class QueueService {
 	}
 
 	@bindThis
-	public createCleanRemoteFilesJob() {
-		return this.objectStorageQueue.add('cleanRemoteFiles', {}, {
+	public createCleanRemoteFilesJob(olderThanSeconds: number = 0, keepFilesInUse: boolean = false) {
+		return this.objectStorageQueue.add('cleanRemoteFiles', {
+			keepFilesInUse,
+			olderThanSeconds,
+		}, {
 			removeOnComplete: {
 				age: 3600 * 24 * 7, // keep up to 7 days
 				count: 30,

packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts

@@ -4,14 +4,17 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import { IsNull, MoreThan, Not } from 'typeorm';
+import { IsNull, MoreThan, Not, Brackets } from 'typeorm';
 import { DI } from '@/di-symbols.js';
 import type { MiDriveFile, DriveFilesRepository } from '@/models/_.js';
+import { MiUser } from '@/models/_.js';
 import type Logger from '@/logger.js';
 import { DriveService } from '@/core/DriveService.js';
 import { bindThis } from '@/decorators.js';
 import { QueueLoggerService } from '../QueueLoggerService.js';
 import type * as Bull from 'bullmq';
+import type { CleanRemoteFilesJobData } from '../types.js';
+import { IdService } from '@/core/IdService.js';
 
 @Injectable()
 export class CleanRemoteFilesProcessorService {
@@ -23,35 +26,54 @@ export class CleanRemoteFilesProcessorService {
 
 		private driveService: DriveService,
 		private queueLoggerService: QueueLoggerService,
+		private idService: IdService,
 	) {
 		this.logger = this.queueLoggerService.logger.createSubLogger('clean-remote-files');
 	}
 
 	@bindThis
-	public async process(job: Bull.Job<Record<string, unknown>>): Promise<void> {
+	public async process(job: Bull.Job<CleanRemoteFilesJobData>): Promise<void> {
 		this.logger.info('Deleting cached remote files...');
 
+		const olderThanTimestamp = Date.now() - (job.data.olderThanSeconds ?? 0) * 1000;
+		const olderThanDate = new Date(olderThanTimestamp);
+		const keepFilesInUse = job.data.keepFilesInUse ?? false;
 		let deletedCount = 0;
 		let cursor: MiDriveFile['id'] | null = null;
 		let errorCount = 0;
 
-		const total = await this.driveFilesRepository.countBy({
-			userHost: Not(IsNull()),
-			isLink: false,
-		});
+		const filesQuery = this.driveFilesRepository.createQueryBuilder('file')
+			.where('file.userHost IS NOT NULL') // remote files
+			.andWhere('file.isLink = FALSE') // cached
+			.andWhere('file.id <= :id', { id: this.idService.gen(olderThanTimestamp) }) // and old
+			.orderBy('file.id', 'ASC');
+
+		if (keepFilesInUse) {
+			filesQuery
+			// are they used as avatar&&c?
+				.leftJoinAndSelect(
+					MiUser, 'fileuser',
+					'fileuser."avatarId"="file"."id" OR fileuser."bannerId"="file"."id" OR fileuser."backgroundId"="file"."id"'
+				)
+				.andWhere(
+					new Brackets((qb) => {
+						qb.where('fileuser.id IS NULL') // not used
+							.orWhere( // or attached to a user
+								new Brackets((qb) => {
+									qb.where('fileuser.lastFetchedAt IS NOT NULL') // weird? maybe this only applies to local users
+										.andWhere('fileuser.lastFetchedAt < :old', { old: olderThanDate }); // old user
+								})
+							);
+					})
+				);
+		}
+
+		const total = await filesQuery.clone().getCount();
 
 		while (true) {
-			const files = await this.driveFilesRepository.find({
-				where: {
-					userHost: Not(IsNull()),
-					isLink: false,
-					...(cursor ? { id: MoreThan(cursor) } : {}),
-				},
-				take: 256,
-				order: {
-					id: 1,
-				},
-			});
+			const thisBatchQuery = filesQuery.clone();
+			if (cursor) thisBatchQuery.andWhere('file.id > :cursor', { cursor });
+			const files = await thisBatchQuery.take(256).getMany();
 
 			if (files.length === 0) {
 				job.updateProgress(100);

packages/backend/src/queue/types.ts

@@ -40,6 +40,11 @@ export type RelationshipJobData = {
 	withReplies?: boolean;
 };
 
+export type CleanRemoteFilesJobData = {
+	keepFilesInUse: boolean;
+	olderThanSeconds: number;
+};
+
 export type DbJobData<T extends keyof DbJobMap> = DbJobMap[T];
 
 export type DbJobMap = {

packages/backend/src/server/api/endpoints/admin/drive/clean-remote-files.ts

@@ -18,7 +18,10 @@ export const meta = {
 
 export const paramDef = {
 	type: 'object',
-	properties: {},
+	properties: {
+		olderThanSeconds: { type: 'number' },
+		keepFilesInUse: { type: 'boolean' },
+	},
 	required: [],
 } as const;
 
@@ -30,7 +33,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 	) {
 		super(meta, paramDef, async (ps, me) => {
 			await this.moderationLogService.log(me, 'clearRemoteFiles', {});
-			await this.queueService.createCleanRemoteFilesJob();
+			await this.queueService.createCleanRemoteFilesJob(
+				ps.olderThanSeconds ?? 0,
+				ps.keepFilesInUse ?? false,
+			);
 		});
 	}
 }

packages/frontend/src/pages/admin/files.vue

@@ -58,14 +58,40 @@ const pagination = {
 	})),
 };
 
-function clear() {
-	os.confirm({
-		type: 'warning',
-		text: i18n.ts.clearCachedFilesConfirm,
-	}).then(({ canceled }) => {
-		if (canceled) return;
+async function clear() {
+	const { canceled, result } = await os.form(i18n.ts.clearCachedFilesOptions.title, {
+		olderThanEnum: {
+			label: i18n.ts.clearCachedFilesOptions.olderThan,
+			type: 'enum',
+			default: 'now',
+			required: true,
+			enum: [
+				{ label: i18n.ts.clearCachedFilesOptions.now, value: 'now' },
+				{ label: i18n.ts.clearCachedFilesOptions.oneWeek, value: 'oneWeek' },
+				{ label: i18n.ts.clearCachedFilesOptions.oneMonth, value: 'oneMonth' },
+				{ label: i18n.ts.clearCachedFilesOptions.oneYear, value: 'oneYear' },
+			],
+		},
+		keepFilesInUse: {
+			label: i18n.ts.clearCachedFilesOptions.keepFilesInUse,
+			description: i18n.ts.clearCachedFilesOptions.keepFilesInUseDescription,
+			type: 'boolean',
+			default: false,
+		},
+	});
 
-		os.apiWithDialog('admin/drive/clean-remote-files', {});
+	if (canceled) return;
+
+	const timesMap = {
+		now: 0,
+		oneWeek: 7 * 86400,
+		oneMonth: 30 * 86400,
+		oneYear: 365 * 86400,
+	};
+
+	await os.apiWithDialog('admin/drive/clean-remote-files', {
+		olderThanSeconds: timesMap[result.olderThanEnum] ?? 0,
+		keepFilesInUse: result.keepFilesInUse,
 	});
 }
 

sharkey-locales/en-US.yml

@@ -636,3 +636,13 @@ rawInfoDescription: "Extended user data in its raw form. These fields are privat
 rawApDescription: "ActivityPub user data in its raw form. These fields are public and accessible to other instances."
 
 signupReason: "Signup Reason"
+
+clearCachedFilesOptions:
+  title: "Delete all cached remote files"
+  olderThan: "Only delete files older than:"
+  now: "now"
+  oneWeek: "one week"
+  oneMonth: "one month"
+  oneYear: "one year"
+  keepFilesInUse: "Don't delete files used as avatars&c"
+  keepFilesInUseDescription: "this option requires more complicated database queries, you may need to increase the value of db.extra.statement_timeout in the configuration file"