diff --git a/common/database.ts b/common/database.ts index ba49963c0..195e1719e 100644 --- a/common/database.ts +++ b/common/database.ts @@ -8,7 +8,7 @@ import { } from "discord.js"; import papaparse from "papaparse"; import { client } from "strife.js"; -import { extractMessageExtremities, getAllMessages } from "../util/discord.js"; +import { getAllMessages, getFilesFromMessage } from "../util/discord.js"; import config from "./config.js"; let timeouts: Record< Snowflake, @@ -37,7 +37,7 @@ for (const message of allDatabaseMessages) { message.author.id === client.user.id ? message : await databaseThread.send({ - ...extractMessageExtremities(message), + files: [...(await getFilesFromMessage(message)).values()], content: message.content, }); } @@ -68,7 +68,7 @@ export default class Database { + databases[this.name] = edited; + const attachment = edited.attachments.first()?.url; const written = @@ -228,9 +230,13 @@ for (const [event, code] of Object.entries({ export async function backupDatabases(channel: TextBasedChannel): Promise { if (process.env.NODE_ENV !== "production") return; - const attachments = Object.values(databases) - .map((database) => database?.attachments.first()) - .filter(Boolean); + const attachments = ( + await Promise.all( + Object.values(databases).map( + async (database) => database && (await getFilesFromMessage(database)).first(), + ), + ) + ).filter(Boolean); await channel.send("# Daily Scradd Database Backup"); while (attachments.length) { diff --git a/modules/board/generate.ts b/modules/board/generate.ts index a8f7e8fb6..059e866cc 100644 --- a/modules/board/generate.ts +++ b/modules/board/generate.ts @@ -9,16 +9,9 @@ import { import { client } from "strife.js"; import config from "../../common/config.js"; import { extractMessageExtremities, messageToEmbed } from "../../util/discord.js"; -import tryCensor, { censor } from "../automod/misc.js"; +import { censor } from "../automod/misc.js"; import { BOARD_EMOJI, type boardDatabase } from "./misc.js"; -/** - * Generate an embed and button to represent a board message with. - * - * @param info - Info to generate a message from. - * @param extraButtons - Extra custom buttons to show. - * @returns The representation of the message. - */ export default async function generateBoardMessage( info: (typeof boardDatabase.data)[number] | Message, extraButtons: { pre?: APIButtonComponent[]; post?: APIButtonComponent[] } = {}, @@ -26,14 +19,8 @@ export default async function generateBoardMessage( const count = info instanceof Message ? info.reactions.resolve(BOARD_EMOJI)?.count || 0 : info.reactions; - /** - * Convert a message to an embed and button representation. - * - * @param message - The message to convert. - * @returns The converted message. - */ async function messageToBoardData(message: Message): Promise { - const { files, embeds } = extractMessageExtremities(message, tryCensor); + const { files, embeds } = await extractMessageExtremities(message, censor); embeds.unshift(await messageToEmbed(message, censor)); return { @@ -75,14 +62,11 @@ export default async function generateBoardMessage( : [...(extraButtons.pre ?? []), ...(extraButtons.post ?? [])]; return { - allowedMentions: { users: [] }, - + ...(await extractMessageExtremities(onBoard, censor)), + content: onBoard.content, components: buttons.length ? [{ type: ComponentType.ActionRow, components: buttons }] : [], - - content: onBoard.content, - embeds: onBoard.embeds.map((oldEmbed) => oldEmbed.data), - files: onBoard.attachments.map((attachment) => attachment), + allowedMentions: { users: [] }, }; } diff --git a/modules/bot/edit.ts b/modules/bot/edit.ts index 032297e70..00711e1e6 100644 --- a/modules/bot/edit.ts +++ b/modules/bot/edit.ts @@ -32,7 +32,7 @@ export default async function editMessage( } const pre = - JSON.stringify(getMessageJSON(interaction.targetMessage), undefined, " ").match( + JSON.stringify(await getMessageJSON(interaction.targetMessage), undefined, " ").match( /.{1,4000}/gsy, ) ?? []; await interaction.showModal({ @@ -91,7 +91,7 @@ export async function submitEdit(interaction: ModalSubmitInteraction, id: string if (!json) return; const message = await interaction.channel?.messages.fetch(id); if (!message) throw new TypeError("Used command in DM!"); - const oldJSON = getMessageJSON(message); + const oldJSON = await getMessageJSON(message); const edited = await message.edit(json).catch(async (error: unknown) => { await interaction.reply({ ephemeral: true, @@ -119,9 +119,11 @@ export async function submitEdit(interaction: ModalSubmitInteraction, id: string .replace(/^-{3} \n\+{3} \n/, ""); const extraDiff = unifiedDiff( JSON.stringify({ ...oldJSON, content: undefined }, undefined, " ").split("\n"), - JSON.stringify({ ...getMessageJSON(edited), content: undefined }, undefined, " ").split( - "\n", - ), + JSON.stringify( + { ...(await getMessageJSON(edited)), content: undefined }, + undefined, + " ", + ).split("\n"), { lineterm: "" }, ) .join("\n") diff --git a/modules/logging/messages.ts b/modules/logging/messages.ts index c8018fdd8..94b679fec 100644 --- a/modules/logging/messages.ts +++ b/modules/logging/messages.ts @@ -11,7 +11,13 @@ import { } from "discord.js"; import config from "../../common/config.js"; import { databaseThread } from "../../common/database.js"; -import { extractMessageExtremities, getBaseChannel, messageToText } from "../../util/discord.js"; +import { + extractMessageExtremities, + getBaseChannel, + isFileExpired, + messageToText, + unsignFiles, +} from "../../util/discord.js"; import { joinWithAnd } from "../../util/text.js"; import log, { LogSeverity, LoggingEmojis, shouldLog } from "./misc.js"; @@ -31,12 +37,18 @@ export async function messageDelete(message: Message | PartialMessage): Promise< const content = !shush && messageToText(message, false); const { embeds, files } = - shush ? { embeds: [], files: [] } : extractMessageExtremities(message); + shush ? + { embeds: [], files: [] } + : await extractMessageExtremities(message, undefined, false); + + const unknownAttachments = message.attachments.filter(isFileExpired); await log( `${LoggingEmojis.MessageDelete} ${message.partial ? "Unknown message" : "Message"}${ message.author ? ` by ${message.author.toString()}` : "" - } in ${message.channel.toString()} (ID: ${message.id}) deleted`, + } in ${message.channel.toString()} (ID: ${message.id}) deleted${ + unknownAttachments.size ? `\n ${unknownAttachments.size} unknown attachment` : "" + }${unknownAttachments.size > 1 ? "s" : ""}`, LogSeverity.ContentEdit, { embeds, @@ -79,7 +91,7 @@ export async function messageDeleteBulk( message.attachments.size ? `${message.attachments.size} attachment` : "" }${message.attachments.size > 1 ? "s" : ""}`; const extremities = - message.embeds.length || message.attachments.size ? + embeds || attachments ? ` (${embeds}${embeds && attachments && ", "}${attachments})` : ""; @@ -180,26 +192,35 @@ export async function messageUpdate( if (!newMessage.author.bot) { const files = []; - const contentDiff = + const diff = !oldMessage.partial && - unifiedDiff(oldMessage.content.split("\n"), newMessage.content.split("\n"), { - lineterm: "", - }) + unifiedDiff( + unsignFiles(oldMessage.content).split("\n"), + unsignFiles(newMessage.content).split("\n"), + { lineterm: "" }, + ) .join("\n") .replace(/^-{3} \n\+{3} \n/, ""); - if (contentDiff) files.push({ content: contentDiff, extension: "diff" }); + if (diff) files.push({ content: diff, extension: "diff" }); + const removedAttachments = oldMessage.attachments.filter( + (file) => !newMessage.attachments.has(file.id), + ); files.push( - ...oldMessage.attachments - .filter((attachment) => !newMessage.attachments.has(attachment.id)) + ...removedAttachments + .filter((file) => !isFileExpired(file)) .map((attachment) => attachment.url), ); if (files.length) { await log( - `${LoggingEmojis.MessageEdit} [${ - oldMessage.partial ? "Unknown message" : "Message" - }](<${newMessage.url}>) by ${newMessage.author.toString()} in ${newMessage.channel.toString()} edited`, + `${LoggingEmojis.MessageEdit} [${oldMessage.partial ? "Unknown message" : "Message"}](<${ + newMessage.url + }>) by ${newMessage.author.toString()} in ${newMessage.channel.toString()} edited${ + removedAttachments.size ? + `\n ${removedAttachments.size} attachment${removedAttachments.size > 1 ? "s" : ""} were removed` + : "" + }`, LogSeverity.ContentEdit, { files }, ); diff --git a/modules/punishments/util.ts b/modules/punishments/util.ts index 312c38128..09becf8b1 100644 --- a/modules/punishments/util.ts +++ b/modules/punishments/util.ts @@ -11,16 +11,14 @@ import { type User, } from "discord.js"; import Database, { allDatabaseMessages } from "../../common/database.js"; -import { GlobalUsersPattern, paginate } from "../../util/discord.js"; +import { GlobalUsersPattern, getFilesFromMessage, paginate } from "../../util/discord.js"; import { convertBase } from "../../util/numbers.js"; -import { gracefulFetch } from "../../util/promises.js"; +import { asyncFilter, gracefulFetch } from "../../util/promises.js"; import { LogSeverity, getLoggingThread } from "../logging/misc.js"; import { EXPIRY_LENGTH } from "./misc.js"; export const strikeDatabase = new Database<{ - /** The ID of the user who was warned. */ user: Snowflake; - /** The time when this strike was issued. */ date: number; id: number | string; count: number; @@ -28,13 +26,13 @@ export const strikeDatabase = new Database<{ }>("strikes"); await strikeDatabase.init(); -const robotopUrl = allDatabaseMessages - .find((message) => message.attachments.first()?.name === "robotop_warns.json") - ?.attachments.first()?.url; -const robotopStrikes = - (robotopUrl && - (await gracefulFetch<{ id: number; mod: Snowflake; reason: string }[]>(robotopUrl))) || - []; +const { value: robotopStrikes = [] } = await asyncFilter(allDatabaseMessages, async (message) => { + const files = await getFilesFromMessage(message); + const file = files.find(({ name }) => name === "robotop_warns.json"); + const strikes = + file && (await gracefulFetch<{ id: number; mod: Snowflake; reason: string }[]>(file.url)); + return strikes ?? false; +}).next(); const strikesCache: Record = {}; diff --git a/modules/roles/misc.ts b/modules/roles/misc.ts index 412bb8197..413cae13c 100644 --- a/modules/roles/misc.ts +++ b/modules/roles/misc.ts @@ -6,8 +6,12 @@ export const CUSTOM_ROLE_PREFIX = "✨ "; const validContentTypes = ["image/jpeg", "image/png", "image/apng", "image/gif", "image/webp"]; /** - * Valid strings: string matching twemojiRegexp, Snowflake of existing server emoji, data: URI, string starting with - * https:// + * Valid strings: + * + * - String matching `twemojiRegexp`. + * - Snowflake of existing server emoji. + * - `data:` URI. + * - String starting with `https://` */ export async function resolveIcon( icon: string, diff --git a/util/discord.ts b/util/discord.ts index 044e21e44..ac5823919 100644 --- a/util/discord.ts +++ b/util/discord.ts @@ -43,130 +43,147 @@ import constants from "../common/constants.js"; import { escapeMessage, stripMarkdown } from "./markdown.js"; import { truncateText } from "./text.js"; -/** - * Extract extremities (embeds, stickers, and attachments) from a message. - * - * @param message - The message to extract extremeties from. - * @param tryCensor - Function to censor bad words. Omit to not censor. - */ -export function extractMessageExtremities( - message: Message, - tryCensor?: (text: string) => false | { censored: string; strikes: number; words: string[][] }, -): { embeds: APIEmbed[]; files: Attachment[] } { - const embeds = [ - ...message.stickers - .filter((sticker) => !tryCensor?.(sticker.name)) - .map( - (sticker): APIEmbed => ({ - color: Colors.Blurple, - image: { url: sticker.url }, - footer: { text: sticker.name }, - }), - ), - ...message.embeds - .filter((embed) => !embed.video && !message.flags.has(MessageFlags.SuppressEmbeds)) - .map(({ data }): APIEmbed => { - const automodInfo = (data.fields ?? []).reduce( - (accumulator, field) => ({ ...accumulator, [field.name]: field.value }), - { - flagged_message_id: message.id, - channel_id: message.channel.id, - keyword: "", - rule_name: "", - }, - ); - - const newEmbed = - message.type === MessageType.AutoModerationAction ? - { - description: data.description ?? message.content, - color: message.member?.displayColor ?? data.color, - author: { - icon_url: (message.member ?? message.author).displayAvatarURL(), - name: (message.member ?? message.author).displayName, - }, - url: messageLink( - message.guild?.id ?? "@me", - automodInfo.channel_id, - automodInfo.flagged_message_id, - ), - footer: { - text: `${automodInfo.keyword && `Keyword: ${automodInfo.keyword}`}${ - automodInfo.keyword && - automodInfo.rule_name && - constants.footerSeperator - }${automodInfo.rule_name && `Rule: ${automodInfo.rule_name}`}`, - }, - } - : { ...data }; - - if (!tryCensor) return newEmbed; - - if (newEmbed.description) { - const censored = tryCensor(newEmbed.description); - if (censored) newEmbed.description = censored.censored; - } - - if (newEmbed.title) { - const censored = tryCensor(newEmbed.title); - if (censored) newEmbed.title = censored.censored; - } - - if (newEmbed.url && tryCensor(newEmbed.url)) newEmbed.url = ""; +export function unsignFiles(content: string): string { + return content.replaceAll( + /https:\/\/(?:cdn|media)\.discordapp\.(?:net|com)\/attachments\/(?:[\w!#$&'()*+,./:;=?@~-]|%\d\d)+/gis, + (match) => { + const url = new URL(match); + return url.origin + url.pathname; + }, + ); +} - if (newEmbed.image?.url && tryCensor(newEmbed.image.url)) - newEmbed.image = undefined; +export function isFileExpired(file: { url: string }): boolean { + const expirey = new URL(file.url).searchParams.get("ex"); + return !!expirey && Number.parseInt(expirey, 16) * 1000 < Date.now(); +} - if (newEmbed.thumbnail?.url && tryCensor(newEmbed.thumbnail.url)) - newEmbed.thumbnail = undefined; +export async function getFilesFromMessage( + message: Message, +): Promise> { + const expired = message.attachments.some(isFileExpired); + if (!expired) return message.attachments; - if (newEmbed.footer?.text) { - const censored = tryCensor(newEmbed.footer.text); - if (censored) newEmbed.footer.text = censored.censored; - } + const fetched = await message.fetch(true); + return fetched.attachments; +} - if (newEmbed.author) { - const censoredName = tryCensor(newEmbed.author.name); - if (censoredName) newEmbed.author.name = censoredName.censored; +export async function extractMessageExtremities( + message: Message, + censor?: (text: string) => string, + forceRefetch = true, +): Promise<{ embeds: APIEmbed[]; files: Attachment[] }> { + const embeds = []; + for (const { data } of message.flags.has(MessageFlags.SuppressEmbeds) ? [] : message.embeds) { + if ( + forceRefetch && + ((data.footer?.icon_url && isFileExpired({ url: data.footer.icon_url })) || + (data.image && isFileExpired(data.image)) || + (data.thumbnail && isFileExpired(data.thumbnail)) || + (data.video?.url && isFileExpired({ url: data.video.url })) || + (data.author?.icon_url && isFileExpired({ url: data.author.icon_url }))) + ) + return await extractMessageExtremities(await message.fetch(true), censor, false); + + const automodInfo = + message.type === MessageType.AutoModerationAction && + (data.fields ?? []).reduce( + (accumulator, field) => ({ ...accumulator, [field.name]: field.value }), + { + flagged_message_id: message.id, + channel_id: message.channel.id, + keyword: "", + rule_name: "", + }, + ); - const censoredUrl = newEmbed.author.url && tryCensor(newEmbed.author.url); - if (censoredUrl) newEmbed.author.url = ""; + const newEmbed = + automodInfo ? + { + ...data, + description: data.description ?? message.content, + color: message.member?.displayColor ?? data.color, + author: { + icon_url: (message.member ?? message.author).displayAvatarURL(), + name: (message.member ?? message.author).displayName, + }, + url: messageLink( + message.guild?.id ?? "@me", + automodInfo.channel_id, + automodInfo.flagged_message_id, + ), + footer: { + text: `${automodInfo.keyword && `Keyword: ${automodInfo.keyword}`}${ + automodInfo.keyword && + automodInfo.rule_name && + constants.footerSeperator + }${automodInfo.rule_name && `Rule: ${automodInfo.rule_name}`}`, + }, + fields: [], } + : { ...data }; + + if (!censor) { + embeds.push(newEmbed); + continue; + } + + newEmbed.title = censor(newEmbed.title ?? ""); + newEmbed.description = censor(newEmbed.description ?? ""); + if (newEmbed.author) newEmbed.author.name = censor(newEmbed.author.name); + if (newEmbed.footer) newEmbed.footer.text = censor(newEmbed.footer.text); + + if (newEmbed.url && newEmbed.url !== censor(newEmbed.url)) newEmbed.url = undefined; + if (newEmbed.author?.url && newEmbed.author.url !== censor(newEmbed.author.url)) + newEmbed.author.url = undefined; + if (newEmbed.thumbnail && newEmbed.thumbnail.url !== censor(newEmbed.thumbnail.url)) + newEmbed.thumbnail = undefined; + if (newEmbed.video?.url && newEmbed.video.url !== censor(newEmbed.video.url)) + newEmbed.video = undefined; + if (newEmbed.image && newEmbed.image.url !== censor(newEmbed.image.url)) + newEmbed.image = undefined; + + newEmbed.fields = (newEmbed.fields ?? []).map((field) => ({ + inline: field.inline, + name: censor(field.name), + value: censor(field.value), + })); + + embeds.push(newEmbed); + } - newEmbed.fields = (newEmbed.fields ?? []).map((field) => { - const censoredName = tryCensor(field.name); - const censoredValue = tryCensor(field.value); - return { - inline: field.inline, - name: censoredName ? censoredName.censored : field.name, - value: censoredValue ? censoredValue.censored : field.value, - }; - }); - - return newEmbed; + const stickers = message.stickers + .filter((sticker) => !censor?.(sticker.name)) + .map( + (sticker): APIEmbed => ({ + color: Colors.Blurple, + image: { url: sticker.url }, + footer: { text: sticker.name }, }), - ]; + ); + + const files = ( + forceRefetch ? + await getFilesFromMessage(message) + : message.attachments.filter((file) => !isFileExpired(file))).values(); - return { embeds: embeds.slice(0, 10), files: [...message.attachments.values()] }; + return { + embeds: [...stickers, ...embeds].slice(0, 10), + files: [...files], + }; } -/** - * Converts a message to a JSON object describing it. - * - * @param message - The message to convert. - * @returns The JSON. - */ -export function getMessageJSON(message: Message): { +export async function getMessageJSON(message: Message): Promise<{ components: APIActionRowComponent[]; content: string; embeds: APIEmbed[]; files: string[]; -} { +}> { return { components: message.components.map((component) => component.toJSON()), content: message.content, embeds: message.embeds.map((embed) => embed.toJSON()), - files: message.attachments.map((attachment) => attachment.url), + files: (await getFilesFromMessage(message)).map((attachment) => attachment.url), } satisfies MessageEditOptions; }