fix: Handle expired files properly

- fix(databases): Ensure the backup is valid
- fix(board): Fix invalid images on board
- fix(board): Recensor when editing board messages
- fix(logging): Don't fetch expired attachments
- fix(logging): Unsign image links in message edit diffs

Signed-off-by: cobalt <61329810+RedGuy12@users.noreply.github.com>

common/database.ts

@@ -8,7 +8,7 @@ import {
 } from "discord.js";
 import papaparse from "papaparse";
 import { client } from "strife.js";
-import { extractMessageExtremities, getAllMessages } from "../util/discord.js";
+import { getAllMessages, getFilesFromMessage } from "../util/discord.js";
 import config from "./config.js";
 let timeouts: Record<
 	Snowflake,
@@ -37,7 +37,7 @@ for (const message of allDatabaseMessages) {
 			message.author.id === client.user.id ?
 				message
 			:	await databaseThread.send({
-					...extractMessageExtremities(message),
+					files: [...(await getFilesFromMessage(message)).values()],
 					content: message.content,
 				});
 	}
@@ -68,7 +68,7 @@ export default class Database<Data extends Record<string, boolean | number | str
 		if (databases[this.name]) await databases[this.name]?.edit(content);
 		this.message = databases[this.name] ||= await databaseThread.send(content);
 
-		const attachment = this.message.attachments.first();
+		const attachment = (await getFilesFromMessage(this.message)).first();
 		if (!attachment) {
 			this.#queueWrite();
 			return;
@@ -160,6 +160,8 @@ export default class Database<Data extends Record<string, boolean | number | str
 					});
 				})
 				.then(async (edited) => {
+					databases[this.name] = edited;
+
 					const attachment = edited.attachments.first()?.url;
 
 					const written =
@@ -228,9 +230,13 @@ for (const [event, code] of Object.entries({
 export async function backupDatabases(channel: TextBasedChannel): Promise<void> {
 	if (process.env.NODE_ENV !== "production") return;
 
-	const attachments = Object.values(databases)
-		.map((database) => database?.attachments.first())
-		.filter(Boolean);
+	const attachments = (
+		await Promise.all(
+			Object.values(databases).map(
+				async (database) => database && (await getFilesFromMessage(database)).first(),
+			),
+		)
+	).filter(Boolean);
 
 	await channel.send("# Daily Scradd Database Backup");
 	while (attachments.length) {

modules/board/generate.ts

@@ -9,31 +9,18 @@ import {
 import { client } from "strife.js";
 import config from "../../common/config.js";
 import { extractMessageExtremities, messageToEmbed } from "../../util/discord.js";
-import tryCensor, { censor } from "../automod/misc.js";
+import { censor } from "../automod/misc.js";
 import { BOARD_EMOJI, type boardDatabase } from "./misc.js";
 
-/**
- * Generate an embed and button to represent a board message with.
- *
- * @param info - Info to generate a message from.
- * @param extraButtons - Extra custom buttons to show.
- * @returns The representation of the message.
- */
 export default async function generateBoardMessage(
 	info: (typeof boardDatabase.data)[number] | Message,
 	extraButtons: { pre?: APIButtonComponent[]; post?: APIButtonComponent[] } = {},
 ): Promise<BaseMessageOptions | undefined> {
 	const count =
 		info instanceof Message ? info.reactions.resolve(BOARD_EMOJI)?.count || 0 : info.reactions;
 
-	/**
-	 * Convert a message to an embed and button representation.
-	 *
-	 * @param message - The message to convert.
-	 * @returns The converted message.
-	 */
 	async function messageToBoardData(message: Message): Promise<BaseMessageOptions> {
-		const { files, embeds } = extractMessageExtremities(message, tryCensor);
+		const { files, embeds } = await extractMessageExtremities(message, censor);
 		embeds.unshift(await messageToEmbed(message, censor));
 
 		return {
@@ -75,14 +62,11 @@ export default async function generateBoardMessage(
 			:	[...(extraButtons.pre ?? []), ...(extraButtons.post ?? [])];
 
 		return {
-			allowedMentions: { users: [] },
-
+			...(await extractMessageExtremities(onBoard, censor)),
+			content: onBoard.content,
 			components:
 				buttons.length ? [{ type: ComponentType.ActionRow, components: buttons }] : [],
-
-			content: onBoard.content,
-			embeds: onBoard.embeds.map((oldEmbed) => oldEmbed.data),
-			files: onBoard.attachments.map((attachment) => attachment),
+			allowedMentions: { users: [] },
 		};
 	}
 

modules/bot/edit.ts

@@ -32,7 +32,7 @@ export default async function editMessage(
 	}
 
 	const pre =
-		JSON.stringify(getMessageJSON(interaction.targetMessage), undefined, "  ").match(
+		JSON.stringify(await getMessageJSON(interaction.targetMessage), undefined, "  ").match(
 			/.{1,4000}/gsy,
 		) ?? [];
 	await interaction.showModal({
@@ -91,7 +91,7 @@ export async function submitEdit(interaction: ModalSubmitInteraction, id: string
 	if (!json) return;
 	const message = await interaction.channel?.messages.fetch(id);
 	if (!message) throw new TypeError("Used command in DM!");
-	const oldJSON = getMessageJSON(message);
+	const oldJSON = await getMessageJSON(message);
 	const edited = await message.edit(json).catch(async (error: unknown) => {
 		await interaction.reply({
 			ephemeral: true,
@@ -119,9 +119,11 @@ export async function submitEdit(interaction: ModalSubmitInteraction, id: string
 		.replace(/^-{3} \n\+{3} \n/, "");
 	const extraDiff = unifiedDiff(
 		JSON.stringify({ ...oldJSON, content: undefined }, undefined, "  ").split("\n"),
-		JSON.stringify({ ...getMessageJSON(edited), content: undefined }, undefined, "  ").split(
-			"\n",
-		),
+		JSON.stringify(
+			{ ...(await getMessageJSON(edited)), content: undefined },
+			undefined,
+			"  ",
+		).split("\n"),
 		{ lineterm: "" },
 	)
 		.join("\n")

modules/logging/messages.ts

@@ -11,7 +11,13 @@ import {
 } from "discord.js";
 import config from "../../common/config.js";
 import { databaseThread } from "../../common/database.js";
-import { extractMessageExtremities, getBaseChannel, messageToText } from "../../util/discord.js";
+import {
+	extractMessageExtremities,
+	getBaseChannel,
+	isFileExpired,
+	messageToText,
+	unsignFiles,
+} from "../../util/discord.js";
 import { joinWithAnd } from "../../util/text.js";
 import log, { LogSeverity, LoggingEmojis, shouldLog } from "./misc.js";
 
@@ -31,12 +37,18 @@ export async function messageDelete(message: Message | PartialMessage): Promise<
 
 	const content = !shush && messageToText(message, false);
 	const { embeds, files } =
-		shush ? { embeds: [], files: [] } : extractMessageExtremities(message);
+		shush ?
+			{ embeds: [], files: [] }
+		:	await extractMessageExtremities(message, undefined, false);
+
+	const unknownAttachments = message.attachments.filter(isFileExpired);
 
 	await log(
 		`${LoggingEmojis.MessageDelete} ${message.partial ? "Unknown message" : "Message"}${
 			message.author ? ` by ${message.author.toString()}` : ""
-		} in ${message.channel.toString()} (ID: ${message.id}) deleted`,
+		} in ${message.channel.toString()} (ID: ${message.id}) deleted${
+			unknownAttachments.size ? `\n ${unknownAttachments.size} unknown attachment` : ""
+		}${unknownAttachments.size > 1 ? "s" : ""}`,
 		LogSeverity.ContentEdit,
 		{
 			embeds,
@@ -79,7 +91,7 @@ export async function messageDeleteBulk(
 						message.attachments.size ? `${message.attachments.size} attachment` : ""
 					}${message.attachments.size > 1 ? "s" : ""}`;
 					const extremities =
-						message.embeds.length || message.attachments.size ?
+						embeds || attachments ?
 							` (${embeds}${embeds && attachments && ", "}${attachments})`
 						:	"";
 
@@ -180,26 +192,35 @@ export async function messageUpdate(
 
 	if (!newMessage.author.bot) {
 		const files = [];
-		const contentDiff =
+		const diff =
 			!oldMessage.partial &&
-			unifiedDiff(oldMessage.content.split("\n"), newMessage.content.split("\n"), {
-				lineterm: "",
-			})
+			unifiedDiff(
+				unsignFiles(oldMessage.content).split("\n"),
+				unsignFiles(newMessage.content).split("\n"),
+				{ lineterm: "" },
+			)
 				.join("\n")
 				.replace(/^-{3} \n\+{3} \n/, "");
-		if (contentDiff) files.push({ content: contentDiff, extension: "diff" });
+		if (diff) files.push({ content: diff, extension: "diff" });
 
+		const removedAttachments = oldMessage.attachments.filter(
+			(file) => !newMessage.attachments.has(file.id),
+		);
 		files.push(
-			...oldMessage.attachments
-				.filter((attachment) => !newMessage.attachments.has(attachment.id))
+			...removedAttachments
+				.filter((file) => !isFileExpired(file))
 				.map((attachment) => attachment.url),
 		);
 
 		if (files.length) {
 			await log(
-				`${LoggingEmojis.MessageEdit} [${
-					oldMessage.partial ? "Unknown message" : "Message"
-				}](<${newMessage.url}>) by ${newMessage.author.toString()} in ${newMessage.channel.toString()} edited`,
+				`${LoggingEmojis.MessageEdit} [${oldMessage.partial ? "Unknown message" : "Message"}](<${
+					newMessage.url
+				}>) by ${newMessage.author.toString()} in ${newMessage.channel.toString()} edited${
+					removedAttachments.size ?
+						`\n ${removedAttachments.size} attachment${removedAttachments.size > 1 ? "s" : ""} were removed`
+					:	""
+				}`,
 				LogSeverity.ContentEdit,
 				{ files },
 			);

modules/punishments/util.ts

@@ -11,30 +11,28 @@ import {
 	type User,
 } from "discord.js";
 import Database, { allDatabaseMessages } from "../../common/database.js";
-import { GlobalUsersPattern, paginate } from "../../util/discord.js";
+import { GlobalUsersPattern, getFilesFromMessage, paginate } from "../../util/discord.js";
 import { convertBase } from "../../util/numbers.js";
-import { gracefulFetch } from "../../util/promises.js";
+import { asyncFilter, gracefulFetch } from "../../util/promises.js";
 import { LogSeverity, getLoggingThread } from "../logging/misc.js";
 import { EXPIRY_LENGTH } from "./misc.js";
 
 export const strikeDatabase = new Database<{
-	/** The ID of the user who was warned. */
 	user: Snowflake;
-	/** The time when this strike was issued. */
 	date: number;
 	id: number | string;
 	count: number;
 	removed: boolean;
 }>("strikes");
 await strikeDatabase.init();
 
-const robotopUrl = allDatabaseMessages
-	.find((message) => message.attachments.first()?.name === "robotop_warns.json")
-	?.attachments.first()?.url;
-const robotopStrikes =
-	(robotopUrl &&
-		(await gracefulFetch<{ id: number; mod: Snowflake; reason: string }[]>(robotopUrl))) ||
-	[];
+const { value: robotopStrikes = [] } = await asyncFilter(allDatabaseMessages, async (message) => {
+	const files = await getFilesFromMessage(message);
+	const file = files.find(({ name }) => name === "robotop_warns.json");
+	const strikes =
+		file && (await gracefulFetch<{ id: number; mod: Snowflake; reason: string }[]>(file.url));
+	return strikes ?? false;
+}).next();
 
 const strikesCache: Record<string, { mod?: string; reason?: string }> = {};
 

modules/roles/misc.ts

@@ -6,8 +6,12 @@ export const CUSTOM_ROLE_PREFIX = "✨ ";
 
 const validContentTypes = ["image/jpeg", "image/png", "image/apng", "image/gif", "image/webp"];
 /**
- * Valid strings: string matching twemojiRegexp, Snowflake of existing server emoji, data: URI, string starting with
- * https://
+ * Valid strings:
+ *
+ * - String matching `twemojiRegexp`.
+ * - Snowflake of existing server emoji.
+ * - `data:` URI.
+ * - String starting with `https://`
  */
 export async function resolveIcon(
 	icon: string,