Move everything to src folder... it had to happen

src/m2d/actions/add-reaction.js

@@ -0,0 +1,31 @@
+// @ts-check
+
+const assert = require("assert").strict
+const Ty = require("../../types")
+
+const passthrough = require("../../passthrough")
+const {discord, sync, db, select} = passthrough
+/** @type {import("../converters/utils")} */
+const utils = sync.require("../converters/utils")
+/** @type {import("../converters/emoji")} */
+const emoji = sync.require("../converters/emoji")
+
+/**
+ * @param {Ty.Event.Outer<Ty.Event.M_Reaction>} event
+ */
+async function addReaction(event) {
+	const channelID = select("channel_room", "channel_id", {room_id: event.room_id}).pluck().get()
+	if (!channelID) return // We just assume the bridge has already been created
+	const messageID = select("event_message", "message_id", {event_id: event.content["m.relates_to"].event_id}, "ORDER BY reaction_part").pluck().get()
+	if (!messageID) return // Nothing can be done if the parent message was never bridged.
+
+	const key = event.content["m.relates_to"].key
+	const discordPreferredEncoding = emoji.encodeEmoji(key, event.content.shortcode)
+	if (!discordPreferredEncoding) return
+
+	await discord.snow.channel.createReaction(channelID, messageID, discordPreferredEncoding) // acting as the discord bot itself
+
+	db.prepare("REPLACE INTO reaction (hashed_event_id, message_id, encoded_emoji) VALUES (?, ?, ?)").run(utils.getEventIDHash(event.event_id), messageID, discordPreferredEncoding)
+}
+
+module.exports.addReaction = addReaction

src/m2d/actions/channel-webhook.js

@@ -0,0 +1,99 @@
+// @ts-check
+
+const assert = require("assert").strict
+const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
+const passthrough = require("../../passthrough")
+const {discord, db, select} = passthrough
+
+/**
+ * Look in the database to find webhook credentials for a channel.
+ * (Note that the credentials may be invalid and need to be re-created if the webhook was interfered with from outside.)
+ * @param {string} channelID
+ * @param {boolean} forceCreate create a new webhook no matter what the database says about the state
+ * @returns id and token for a webhook for that channel
+ */
+async function ensureWebhook(channelID, forceCreate = false) {
+	if (!forceCreate) {
+		const row = select("webhook", ["webhook_id", "webhook_token"], {channel_id: channelID}).get()
+		if (row) {
+			return {
+				id: row.webhook_id,
+				token: row.webhook_token,
+				created: false
+			}
+		}
+	}
+
+	// If we got here, we need to create a new webhook.
+	const webhook = await discord.snow.webhook.createWebhook(channelID, {name: "Out Of Your Element: Matrix Bridge"})
+	assert(webhook.token)
+	db.prepare("REPLACE INTO webhook (channel_id, webhook_id, webhook_token) VALUES (?, ?, ?)").run(channelID, webhook.id, webhook.token)
+	return {
+		id: webhook.id,
+		token: webhook.token,
+		created: true
+	}
+}
+
+/**
+ * @param {string} channelID
+ * @param {(webhook: import("../../types").WebhookCreds) => Promise<T>} callback
+ * @returns Promise<T>
+ * @template T
+ */
+async function withWebhook(channelID, callback) {
+	const webhook = await ensureWebhook(channelID, false)
+	return callback(webhook).catch(async e => {
+		if (e.message === `{"message": "Unknown Webhook", "code": 10015}`) { // pathetic error handling from SnowTransfer
+			// Our webhook is gone. Maybe somebody deleted it, or removed and re-added OOYE from the guild.
+			const newWebhook = await ensureWebhook(channelID, true)
+			return callback(newWebhook) // not caught; if the error happens again just throw it instead of looping
+		}
+
+		throw e
+	})
+}
+
+/**
+ * @param {string} channelID
+ * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]}} data
+ * @param {string} [threadID]
+ */
+async function sendMessageWithWebhook(channelID, data, threadID) {
+	const result = await withWebhook(channelID, async webhook => {
+		return discord.snow.webhook.executeWebhook(webhook.id, webhook.token, data, {wait: true, thread_id: threadID})
+	})
+	return result
+}
+
+/**
+ * @param {string} channelID
+ * @param {string} messageID
+ * @param {DiscordTypes.RESTPatchAPIWebhookWithTokenMessageJSONBody & {files?: {name: string, file: Buffer | Readable}[]}} data
+ * @param {string} [threadID]
+ */
+async function editMessageWithWebhook(channelID, messageID, data, threadID) {
+	const result = await withWebhook(channelID, async webhook => {
+		return discord.snow.webhook.editWebhookMessage(webhook.id, webhook.token, messageID, {...data, thread_id: threadID})
+	})
+	return result
+}
+
+/**
+ * @param {string} channelID
+ * @param {string} messageID
+ * @param {string} [threadID]
+ */
+async function deleteMessageWithWebhook(channelID, messageID, threadID) {
+	const result = await withWebhook(channelID, async webhook => {
+		return discord.snow.webhook.deleteWebhookMessage(webhook.id, webhook.token, messageID, threadID)
+	})
+	return result
+}
+
+module.exports.ensureWebhook = ensureWebhook
+module.exports.withWebhook = withWebhook
+module.exports.sendMessageWithWebhook = sendMessageWithWebhook
+module.exports.editMessageWithWebhook = editMessageWithWebhook
+module.exports.deleteMessageWithWebhook = deleteMessageWithWebhook

src/m2d/actions/emoji-sheet.js

@@ -0,0 +1,36 @@
+// @ts-check
+
+const assert = require("assert")
+const fetch = require("node-fetch").default
+
+const utils = require("../converters/utils")
+const {sync} = require("../../passthrough")
+
+/** @type {import("../converters/emoji-sheet")} */
+const emojiSheetConverter = sync.require("../converters/emoji-sheet")
+
+/**
+ * Downloads the emoji from the web and converts to uncompressed PNG data.
+ * @param {string} mxc a single mxc:// URL
+ * @returns {Promise<Buffer | undefined>} uncompressed PNG data, or undefined if the downloaded emoji is not valid
+ */
+async function getAndConvertEmoji(mxc) {
+	const abortController = new AbortController()
+
+	const url = utils.getPublicUrlForMxc(mxc)
+	assert(url)
+
+	/** @type {import("node-fetch").Response} */
+	// If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing.
+	// If we were using connection pooling, we would be forced to download the entire GIF.
+	// So we set no agent to ensure we are not connection pooling.
+	// @ts-ignore the signal is slightly different from the type it wants (still works fine)
+	const res = await fetch(url, {agent: false, signal: abortController.signal})
+	return emojiSheetConverter.convertImageStream(res.body, () => {
+		abortController.abort()
+		res.body.pause()
+		res.body.emit("end")
+	})
+}
+
+module.exports.getAndConvertEmoji = getAndConvertEmoji

src/m2d/actions/redact.js

@@ -0,0 +1,43 @@
+// @ts-check
+
+const assert = require("assert").strict
+const Ty = require("../../types")
+
+const passthrough = require("../../passthrough")
+const {discord, sync, db, select, from} = passthrough
+/** @type {import("../converters/utils")} */
+const utils = sync.require("../converters/utils")
+
+/**
+ * @param {Ty.Event.Outer_M_Room_Redaction} event
+ */
+async function deleteMessage(event) {
+	const rows = from("event_message").join("message_channel", "message_id").select("channel_id", "message_id").where({event_id: event.redacts}).all()
+	db.prepare("DELETE FROM event_message WHERE event_id = ?").run(event.event_id)
+	for (const row of rows) {
+		db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(row.message_id)
+		await discord.snow.channel.deleteMessage(row.channel_id, row.message_id, event.content.reason)
+	}
+}
+
+/**
+ * @param {Ty.Event.Outer_M_Room_Redaction} event
+ */
+async function removeReaction(event) {
+	const hash = utils.getEventIDHash(event.redacts)
+	const row = from("reaction").join("message_channel", "message_id").select("channel_id", "message_id", "encoded_emoji").where({hashed_event_id: hash}).get()
+	if (!row) return
+	await discord.snow.channel.deleteReactionSelf(row.channel_id, row.message_id, row.encoded_emoji)
+	db.prepare("DELETE FROM reaction WHERE hashed_event_id = ?").run(hash)
+}
+
+/**
+ * Try everything that could possibly be redacted.
+ * @param {Ty.Event.Outer_M_Room_Redaction} event
+ */
+async function handle(event) {
+	await deleteMessage(event)
+	await removeReaction(event)
+}
+
+module.exports.handle = handle

src/m2d/actions/send-event.js

@@ -0,0 +1,147 @@
+// @ts-check
+
+const Ty = require("../../types")
+const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
+const assert = require("assert").strict
+const crypto = require("crypto")
+const fetch = require("node-fetch").default
+const passthrough = require("../../passthrough")
+const {sync, discord, db, select} = passthrough
+
+/** @type {import("./channel-webhook")} */
+const channelWebhook = sync.require("./channel-webhook")
+/** @type {import("../converters/event-to-message")} */
+const eventToMessage = sync.require("../converters/event-to-message")
+/** @type {import("../../matrix/api")}) */
+const api = sync.require("../../matrix/api")
+/** @type {import("../../d2m/actions/register-user")} */
+const registerUser = sync.require("../../d2m/actions/register-user")
+/** @type {import("../../d2m/actions/edit-message")} */
+const editMessage = sync.require("../../d2m/actions/edit-message")
+/** @type {import("../actions/emoji-sheet")} */
+const emojiSheet = sync.require("../actions/emoji-sheet")
+
+/**
+ * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer | Readable})[]}} message
+ * @returns {Promise<DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]}>}
+ */
+async function resolvePendingFiles(message) {
+	if (!message.pendingFiles) return message
+	const files = await Promise.all(message.pendingFiles.map(async p => {
+		if ("buffer" in p) {
+			return {
+				name: p.name,
+				file: p.buffer
+			}
+		}
+		if ("key" in p) {
+			// Encrypted file
+			const d = crypto.createDecipheriv("aes-256-ctr", Buffer.from(p.key, "base64url"), Buffer.from(p.iv, "base64url"))
+			// @ts-ignore
+			fetch(p.url).then(res => res.body.pipe(d))
+			return {
+				name: p.name,
+				file: d
+			}
+		} else {
+			// Unencrypted file
+			/** @type {Readable} */ // @ts-ignore
+			const body = await fetch(p.url).then(res => res.body)
+			return {
+				name: p.name,
+				file: body
+			}
+		}
+	}))
+	const newMessage = {
+		...message,
+		files: files.concat(message.files || [])
+	}
+	delete newMessage.pendingFiles
+	return newMessage
+}
+
+/** @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker} event */
+async function sendEvent(event) {
+	const row = select("channel_room", ["channel_id", "thread_parent"], {room_id: event.room_id}).get()
+	if (!row) return // allow the bot to exist in unbridged rooms, just don't do anything with it
+	let channelID = row.channel_id
+	let threadID = undefined
+	if (row.thread_parent) {
+		threadID = channelID
+		channelID = row.thread_parent // it's the thread's parent... get with the times...
+	}
+	// @ts-ignore
+	const guildID = discord.channels.get(channelID).guild_id
+	const guild = discord.guilds.get(guildID)
+	assert(guild)
+
+	// no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it
+
+	let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow, fetch, mxcDownloader: emojiSheet.getAndConvertEmoji})
+
+	messagesToEdit = await Promise.all(messagesToEdit.map(async e => {
+		e.message = await resolvePendingFiles(e.message)
+		return e
+	}))
+	messagesToSend = await Promise.all(messagesToSend.map(message => {
+		return resolvePendingFiles(message)
+	}))
+
+	let eventPart = 0 // 0 is primary, 1 is supporting
+	const pendingEdits = []
+
+	/** @type {DiscordTypes.APIMessage[]} */
+	const messageResponses = []
+	for (const data of messagesToEdit) {
+		const messageResponse = await channelWebhook.editMessageWithWebhook(channelID, data.id, data.message, threadID)
+		eventPart = 1
+		messageResponses.push(messageResponse)
+	}
+
+	for (const id of messagesToDelete) {
+		db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(id)
+		db.prepare("DELETE FROM event_message WHERE message_id = ?").run(id)
+		await channelWebhook.deleteMessageWithWebhook(channelID, id, threadID)
+	}
+
+	for (const message of messagesToSend) {
+		const reactionPart = messagesToEdit.length === 0 && message === messagesToSend[messagesToSend.length - 1] ? 0 : 1
+		const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message, threadID)
+		db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(messageResponse.id, threadID || channelID)
+		db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content["msgtype"] || null, messageResponse.id, eventPart, reactionPart) // source 0 = matrix
+
+		eventPart = 1
+		messageResponses.push(messageResponse)
+
+		/*
+			If the Discord system has a cached link preview embed for one of the links just sent,
+			it will be instantly added as part of `embeds` and there won't be a MESSAGE_UPDATE.
+			To reflect the generated embed back to Matrix, we pretend the message was updated right away.
+		*/
+		const sentEmbedsCount = message.embeds?.length || 0
+		if (messageResponse.embeds.length > sentEmbedsCount) {
+			// not awaiting here because requests to Matrix shouldn't block requests to Discord
+			pendingEdits.push(() =>
+				// @ts-ignore this is a valid message edit payload
+				editMessage.editMessage({
+					id: messageResponse.id,
+					channel_id: messageResponse.channel_id,
+					guild_id: guild.id,
+					embeds: messageResponse.embeds
+				}, guild, null)
+			)
+		}
+	}
+
+	for (const user of ensureJoined) {
+		registerUser.ensureSimJoined(user, event.room_id)
+	}
+
+	await Promise.all(pendingEdits.map(f => f())) // `await` will propagate any errors during editing
+
+	return messageResponses
+}
+
+module.exports.sendEvent = sendEvent

src/m2d/converters/emoji-sheet.js

@@ -0,0 +1,114 @@
+// @ts-check
+
+const assert = require("assert").strict
+const {pipeline} = require("stream").promises
+const sharp = require("sharp")
+const {GIFrame} = require("@cloudrac3r/giframe")
+const {PNG} = require("@cloudrac3r/pngjs")
+const streamMimeType = require("stream-mime-type")
+
+const SIZE = 48
+const RESULT_WIDTH = 400
+const IMAGES_ACROSS = Math.floor(RESULT_WIDTH / SIZE)
+
+/**
+ * Composite a bunch of Matrix emojis into a kind of spritesheet image to upload to Discord.
+ * @param {string[]} mxcs mxc URLs, in order
+ * @param {(mxc: string) => Promise<Buffer | undefined>} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock.
+ * @returns {Promise<Buffer>} PNG image
+ */
+async function compositeMatrixEmojis(mxcs, mxcDownloader) {
+	const buffers = await Promise.all(mxcs.map(mxcDownloader))
+
+	// Calculate the size of the final composited image
+	const totalWidth = Math.min(buffers.length, IMAGES_ACROSS) * SIZE
+	const imagesDown = Math.ceil(buffers.length / IMAGES_ACROSS)
+	const totalHeight = imagesDown * SIZE
+	const comp = []
+	let left = 0, top = 0
+	for (const buffer of buffers) {
+		if (Buffer.isBuffer(buffer)) {
+			// Composite the current buffer into the sprite sheet
+			comp.push({left, top, input: buffer})
+			// The next buffer should be placed one slot to the right
+			left += SIZE
+			// If we're out of space to fit the entire next buffer there, wrap to the next line
+			if (left + SIZE > RESULT_WIDTH) {
+				left = 0
+				top += SIZE
+			}
+		}
+	}
+
+	const output = await sharp({create: {width: totalWidth, height: totalHeight, channels: 4, background: {r: 0, g: 0, b: 0, alpha: 0}}})
+		.composite(comp)
+		.png()
+		.toBuffer({resolveWithObject: true})
+	return output.data
+}
+
+/**
+ * @param {import("node-fetch").Response["body"]} streamIn
+ * @param {() => any} stopStream
+ * @returns {Promise<Buffer | undefined>} Uncompressed PNG image
+ */
+async function convertImageStream(streamIn, stopStream) {
+	const {stream, mime} = await streamMimeType.getMimeType(streamIn)
+	assert(["image/png", "image/jpeg", "image/webp", "image/gif", "image/apng"].includes(mime), `Mime type ${mime} is impossible for emojis`)
+
+	try {
+		if (mime === "image/png" || mime === "image/jpeg" || mime === "image/webp") {
+			/** @type {{info: sharp.OutputInfo, buffer: Buffer}} */
+			const result = await new Promise((resolve, reject) => {
+				const transformer = sharp()
+					.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
+					.png({compressionLevel: 0})
+					.toBuffer((err, buffer, info) => {
+						/* c8 ignore next */
+						if (err) return reject(err)
+						resolve({info, buffer})
+					})
+				pipeline(
+					stream,
+					transformer
+				)
+			})
+			return result.buffer
+
+		} else if (mime === "image/gif") {
+			const giframe = new GIFrame(0)
+			stream.on("data", chunk => {
+				giframe.feed(chunk)
+			})
+			const frame = await giframe.getFrame()
+			const pixels = Uint8Array.from(frame.pixels)
+			stopStream()
+
+			const buffer = await sharp(pixels, {raw: {width: frame.width, height: frame.height, channels: 4}})
+				.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
+				.png({compressionLevel: 0})
+				.toBuffer({resolveWithObject: true})
+			return buffer.data
+
+		} else if (mime === "image/apng") {
+			const png = new PNG({maxFrames: 1})
+			// @ts-ignore
+			stream.pipe(png)
+			/** @type {Buffer} */ // @ts-ignore
+			const frame = await new Promise(resolve => png.on("parsed", resolve))
+			stopStream()
+
+			const buffer = await sharp(frame, {raw: {width: png.width, height: png.height, channels: 4}})
+				.resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}})
+				.png({compressionLevel: 0})
+				.toBuffer({resolveWithObject: true})
+			return buffer.data
+
+		}
+	} finally {
+		stopStream()
+	}
+}
+
+module.exports.compositeMatrixEmojis = compositeMatrixEmojis
+module.exports.convertImageStream = convertImageStream

src/m2d/converters/emoji-sheet.test.js

@@ -0,0 +1,58 @@
+const {test} = require("supertape")
+const {convertImageStream} = require("./emoji-sheet")
+const fs = require("fs")
+const {Transform} = require("stream").Transform
+
+/* c8 ignore next 7 */
+function slow() {
+	if (process.argv.includes("--slow")) {
+		return test
+	} else {
+		return test.skip
+	}
+}
+
+class Meter extends Transform {
+	bytes = 0
+
+	_transform(chunk, encoding, cb) {
+		this.bytes += chunk.length
+		this.push(chunk)
+		cb()
+	}
+}
+
+/**
+ * @param {import("supertape").Test} t
+ * @param {string} path
+ * @param {number} totalSize
+ * @param {number => boolean} sizeCheck
+ */
+async function runSingleTest(t, path, totalSize, sizeCheck) {
+	const file = fs.createReadStream(path)
+	const meter = new Meter()
+	const p = file.pipe(meter)
+	const result = await convertImageStream(p, () => {
+		file.pause()
+		file.emit("end")
+	})
+	t.equal(result.subarray(1, 4).toString("ascii"), "PNG", `test that this is a PNG file: ${result.toString("base64").slice(0, 100)}`)
+	/* c8 ignore next 5 */
+	if (sizeCheck(meter.bytes)) {
+		t.pass("read the correct amount of the file")
+	} else {
+		t.fail(`read too much or too little of the file, read: ${meter.bytes}, total: ${totalSize}`)
+	}
+}
+
+slow()("emoji-sheet: only partial file is read for APNG", async t => {
+	await runSingleTest(t, "test/res/butterfly.png", 2438998, n => n < 2438998 / 4) // should download less than 25% of the file
+})
+
+slow()("emoji-sheet: only partial file is read for GIF", async t => {
+	await runSingleTest(t, "test/res/butterfly.gif", 781223, n => n < 781223 / 4) // should download less than 25% of the file
+})
+
+slow()("emoji-sheet: entire file is read for static PNG", async t => {
+	await runSingleTest(t, "test/res/RLMgJGfgTPjIQtvvWZsYjhjy.png", 11301, n => n === 11301) // should download entire file
+})

src/m2d/converters/emoji.js

@@ -0,0 +1,57 @@
+// @ts-check
+
+const assert = require("assert").strict
+const Ty = require("../../types")
+
+const passthrough = require("../../passthrough")
+const {sync, select} = passthrough
+
+/**
+ * @param {string} input
+ * @param {string | null | undefined} shortcode
+ * @returns {string?}
+ */
+function encodeEmoji(input, shortcode) {
+	let discordPreferredEncoding
+	if (input.startsWith("mxc://")) {
+		// Custom emoji
+		let row = select("emoji", ["emoji_id", "name"], {mxc_url: input}).get()
+		if (!row && shortcode) {
+			// Use the name to try to find a known emoji with the same name.
+			const name = shortcode.replace(/^:|:$/g, "")
+			row = select("emoji", ["emoji_id", "name"], {name: name}).get()
+		}
+		if (!row) {
+			// We don't have this emoji and there's no realistic way to just-in-time upload a new emoji somewhere.
+			// Sucks!
+			return null
+		}
+		// Cool, we got an exact or a candidate emoji.
+		discordPreferredEncoding = encodeURIComponent(`${row.name}:${row.emoji_id}`)
+	} else {
+		// Default emoji
+		// https://github.com/discord/discord-api-docs/issues/2723#issuecomment-807022205 ????????????
+		const encoded = encodeURIComponent(input)
+		const encodedTrimmed = encoded.replace(/%EF%B8%8F/g, "")
+
+		const forceTrimmedList = [
+			"%F0%9F%91%8D", // 👍
+			"%F0%9F%91%8E", // 👎️
+			"%E2%AD%90", // ⭐
+			"%F0%9F%90%88", // 🐈
+			"%E2%9D%93", // ❓
+			"%F0%9F%8F%86", // 🏆️
+			"%F0%9F%93%9A", // 📚️
+		]
+
+		discordPreferredEncoding =
+			( forceTrimmedList.includes(encodedTrimmed) ? encodedTrimmed
+			: encodedTrimmed !== encoded && [...input].length === 2 ? encoded
+			: encodedTrimmed)
+
+		console.log("add reaction from matrix:", input, encoded, encodedTrimmed, "chosen:", discordPreferredEncoding)
+	}
+	return discordPreferredEncoding
+}
+
+module.exports.encodeEmoji = encodeEmoji

src/m2d/converters/event-to-message.js

@@ -0,0 +1,856 @@
+// @ts-check
+
+const Ty = require("../../types")
+const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
+const chunk = require("chunk-text")
+const TurndownService = require("@cloudrac3r/turndown")
+const domino = require("domino")
+const assert = require("assert").strict
+const entities = require("entities")
+
+const passthrough = require("../../passthrough")
+const {sync, db, discord, select, from} = passthrough
+/** @type {import("../converters/utils")} */
+const mxUtils = sync.require("../converters/utils")
+/** @type {import("../../discord/utils")} */
+const dUtils = sync.require("../../discord/utils")
+/** @type {import("../../matrix/file")} */
+const file = sync.require("../../matrix/file")
+/** @type {import("./emoji-sheet")} */
+const emojiSheet = sync.require("./emoji-sheet")
+
+/** @type {[RegExp, string][]} */
+const markdownEscapes = [
+	[/\\/g, '\\\\'],
+	[/\*/g, '\\*'],
+	[/^-/g, '\\-'],
+	[/^\+ /g, '\\+ '],
+	[/^(=+)/g, '\\$1'],
+	[/^(#{1,6}) /g, '\\$1 '],
+	[/`/g, '\\`'],
+	[/^~~~/g, '\\~~~'],
+	[/\[/g, '\\['],
+	[/\]/g, '\\]'],
+	[/^>/g, '\\>'],
+	[/_/g, '\\_'],
+	[/^(\d+)\. /g, '$1\\. ']
+	/*
+		Strikethrough is deliberately not escaped. Usually when Matrix users type ~~ it's not because they wanted to send ~~,
+		it's because they wanted strikethrough and it didn't work because their client doesn't support it.
+		As bridge developers, we can choose between "messages should look as similar as possible" vs "it was most likely intended to be strikethrough".
+		I went with the latter. Even though the appearance doesn't match, I'd rather it displayed as originally intended for 80% of the readers than for 0%.
+	*/
+]
+
+const turndownService = new TurndownService({
+	hr: "----",
+	headingStyle: "atx",
+	preformattedCode: true,
+	codeBlockStyle: "fenced"
+})
+
+/**
+ * Markdown characters in the HTML content need to be escaped, though take care not to escape the middle of bare links
+ * @param {string} string
+ */
+// @ts-ignore bad type from turndown
+turndownService.escape = function (string) {
+	return string.replace(/\s+|\S+/g, part => { // match chunks of spaces or non-spaces
+		if (part.match(/\s/)) return part // don't process spaces
+
+		if (part.match(/^https?:\/\//)) {
+			return part
+		} else {
+			return markdownEscapes.reduce(function (accumulator, escape) {
+				return accumulator.replace(escape[0], escape[1])
+			}, part)
+		}
+	})
+}
+
+turndownService.remove("mx-reply")
+
+turndownService.addRule("strikethrough", {
+	filter: ["del", "s"],
+	replacement: function (content) {
+		return "~~" + content + "~~"
+	}
+})
+
+turndownService.addRule("underline", {
+	filter: ["u"],
+	replacement: function (content) {
+		return "__" + content + "__"
+	}
+})
+
+turndownService.addRule("blockquote", {
+	filter: "blockquote",
+	replacement: function (content) {
+		content = content.replace(/^\n+|\n+$/g, "")
+		content = content.replace(/^/gm, "> ")
+		return content
+	}
+})
+
+turndownService.addRule("spoiler", {
+	filter: function (node, options) {
+		return node.tagName === "SPAN" && node.hasAttribute("data-mx-spoiler")
+	},
+
+	replacement: function (content, node) {
+		if (node.getAttribute("data-mx-spoiler")) {
+			// escape parentheses so it can't become a link
+			return `\\(${node.getAttribute("data-mx-spoiler")}\\) ||${content}||`
+		}
+		return `||${content}||`
+	}
+})
+
+turndownService.addRule("inlineLink", {
+	filter: function (node, options) {
+		return (
+			node.nodeName === "A" &&
+			node.getAttribute("href")
+		)
+	},
+
+	replacement: function (content, node) {
+		if (node.getAttribute("data-user-id")) {
+			const user_id = node.getAttribute("data-user-id")
+			const row = select("sim_proxy", ["displayname", "proxy_owner_id"], {user_id}).get()
+			if (row) {
+				return `**@${row.displayname}** (<@${row.proxy_owner_id}>)`
+			} else {
+				return `<@${user_id}>`
+			}
+		}
+		if (node.getAttribute("data-message-id")) return `https://discord.com/channels/${node.getAttribute("data-guild-id")}/${node.getAttribute("data-channel-id")}/${node.getAttribute("data-message-id")}`
+		if (node.getAttribute("data-channel-id")) return `<#${node.getAttribute("data-channel-id")}>`
+		const href = node.getAttribute("href")
+		content = content.replace(/ @.*/, "")
+		if (href === content) return href
+		if (decodeURIComponent(href).startsWith("https://matrix.to/#/@") && content[0] !== "@") content = "@" + content
+		return "[" + content + "](" + href + ")"
+	}
+})
+
+turndownService.addRule("listItem", {
+	filter: "li",
+	replacement: function (content, node, options) {
+		content = content
+		.replace(/^\n+/, "") // remove leading newlines
+		.replace(/\n+$/, "\n") // replace trailing newlines with just a single one
+		.replace(/\n/gm, "\n  ") // indent
+		var prefix = options.bulletListMarker + " "
+		var parent = node.parentNode
+		if (parent.nodeName === "OL") {
+			var start = parent.getAttribute("start")
+			var index = Array.prototype.indexOf.call(parent.children, node)
+			prefix = (start ? Number(start) + index : index + 1) + ". "
+		}
+		return prefix + content + (node.nextSibling && !/\n$/.test(content) ? "\n" : "")
+	}
+})
+
+/** @type {string[]} SPRITE SHEET EMOJIS FEATURE: mxc urls for the currently processing message */
+let endOfMessageEmojis = []
+turndownService.addRule("emoji", {
+	filter: function (node, options) {
+		if (node.nodeName !== "IMG" || !node.hasAttribute("data-mx-emoticon") || !node.getAttribute("src") || !node.getAttribute("title")) return false
+		return true
+	},
+
+	replacement: function (content, node) {
+		const mxcUrl = node.getAttribute("src")
+		const guessedName = node.getAttribute("title").replace(/^:|:$/g, "")
+		return convertEmoji(mxcUrl, guessedName, true, true)
+	}
+})
+
+turndownService.addRule("fencedCodeBlock", {
+	filter: function (node, options) {
+		return (
+			options.codeBlockStyle === "fenced" &&
+			node.nodeName === "PRE" &&
+			node.firstChild &&
+			node.firstChild.nodeName === "CODE"
+		)
+	},
+	replacement: function (content, node, options) {
+		const className = node.firstChild.getAttribute("class") || ""
+		const language = (className.match(/language-(\S+)/) || [null, ""])[1]
+		const code = node.firstChild
+		const visibleCode = getCodeContent(code)
+
+		var fence = "```"
+
+		return (
+			fence + language + "\n" +
+			visibleCode +
+			"\n" + fence
+		)
+	}
+})
+
+/** @param {{ childNodes: Node[]; }} preCode the <code> directly inside the <pre> */
+function getCodeContent(preCode) {
+	return preCode.childNodes.map(c => c.nodeName === "BR" ? "\n" : c.textContent).join("").replace(/\n*$/g, "")
+}
+
+/**
+ * @param {string | null} mxcUrl
+ * @param {string | null} nameForGuess without colons
+ * @param {boolean} allowSpriteSheetIndicator
+ * @param {boolean} allowLink
+ * @returns {string} discord markdown that represents the custom emoji in some form
+ */
+function convertEmoji(mxcUrl, nameForGuess, allowSpriteSheetIndicator, allowLink) {
+	// Get the known emoji from the database.
+	if (mxcUrl) var row = select("emoji", ["emoji_id", "name", "animated"], {mxc_url: mxcUrl}).get()
+	// Now we have to search all servers to see if we're able to send this emoji.
+	if (row) {
+		const found = [...discord.guilds.values()].find(g => g.emojis.find(e => e.id === row?.emoji_id))
+		if (!found) row = null
+	}
+	// Or, if we don't have an emoji right now, we search for the name instead.
+	if (!row && nameForGuess) {
+		const nameForGuessLower = nameForGuess.toLowerCase()
+		for (const guild of discord.guilds.values()) {
+			/** @type {{name: string, id: string, animated: number}[]} */
+			// @ts-ignore
+			const emojis = guild.emojis
+			const found = emojis.find(e => e.name?.toLowerCase() === nameForGuessLower)
+			if (found) {
+				row = {
+					animated: found.animated,
+					emoji_id: found.id,
+					name: found.name
+				}
+				break
+			}
+		}
+	}
+	if (row) {
+		// We know an emoji, and we can use it
+		const animatedChar = row.animated ? "a" : ""
+		return `<${animatedChar}:${row.name}:${row.emoji_id}>`
+	} else if (allowSpriteSheetIndicator && mxcUrl && endOfMessageEmojis.includes(mxcUrl)) {
+		// We can't locate or use a suitable emoji. After control returns, it will rewind over this, delete this section, and upload the emojis as a sprite sheet.
+		return `<::>`
+	} else if (allowLink && mxcUrl && nameForGuess) {
+		// We prefer not to upload this as a sprite sheet because the emoji is not at the end of the message, it is in the middle.
+		return `[:${nameForGuess}:](${mxUtils.getPublicUrlForMxc(mxcUrl)})`
+	} else if (nameForGuess) {
+		return `:${nameForGuess}:`
+	} else {
+		return ""
+	}
+}
+
+/**
+ * @param {string} roomID
+ * @param {string} mxid
+ * @returns {Promise<{displayname?: string?, avatar_url?: string?}>}
+ */
+async function getMemberFromCacheOrHomeserver(roomID, mxid, api) {
+	const row = select("member_cache", ["displayname", "avatar_url"], {room_id: roomID, mxid}).get()
+	if (row) return row
+	return api.getStateEvent(roomID, "m.room.member", mxid).then(event => {
+		db.prepare("REPLACE INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?)").run(roomID, mxid, event?.displayname || null, event?.avatar_url || null)
+		return event
+	}).catch(() => {
+		return {displayname: null, avatar_url: null}
+	})
+}
+
+/**
+ * Splits a display name into one chunk containing <=80 characters (80 being how many characters Discord allows for the name of a webhook),
+ * and another chunk containing the rest of the characters. Splits on whitespace if possible.
+ * These chunks, respectively, go in the display name, and at the top of the message.
+ * If the second part isn't empty, it'll also contain boldening markdown and a line break at the end, so that regardless of its value it
+ * can be prepended to the message content as-is.
+ * @summary Splits too-long Matrix names into a display name chunk and a message content chunk.
+ * @param  {string} displayName - The Matrix side display name to chop up.
+ * @returns {[string, string]} [shortened display name, display name runoff]
+ */
+function splitDisplayName(displayName) {
+	/** @type {string[]} */
+	let displayNameChunks = chunk(displayName, 80)
+
+	if (displayNameChunks.length === 1) {
+		return [displayName, ""]
+	} else {
+		const displayNamePreRunoff = displayNameChunks[0]
+		// displayNameRunoff is a slice of the original rather than a concatenation of the rest of the chunks in order to preserve whatever whitespace it was broken on.
+		const displayNameRunoff = `**${displayName.slice(displayNamePreRunoff.length + 1)}**\n`
+
+		return [displayNamePreRunoff, displayNameRunoff]
+	}
+}
+
+/**
+ * Convert a Matrix user ID into a Discord user ID for mentioning, where if the user is a PK proxy, it will mention the proxy owner.
+ * @param {string} mxid
+ */
+function getUserOrProxyOwnerID(mxid) {
+	const row = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({mxid}).get()
+	if (!row) return null
+	return row.proxy_owner_id || row.user_id
+}
+
+/**
+ * At the time of this executing, we know what the end of message emojis are, and we know that at least one of them is unknown.
+ * This function will strip them from the content and generate the correct pending file of the sprite sheet.
+ * @param {string} content
+ * @param {{id: string, name: string}[]} attachments
+ * @param {({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer})[]} pendingFiles
+ * @param {(mxc: string) => Promise<Buffer | undefined>} mxcDownloader function that will download the mxc URLs and convert to uncompressed PNG data. use `getAndConvertEmoji` or a mock.
+ */
+async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, mxcDownloader) {
+	if (!content.includes("<::>")) return content // No unknown emojis, nothing to do
+	// Remove known and unknown emojis from the end of the message
+	const r = /<a?:[a-zA-Z0-9_]*:[0-9]*>\s*$/
+	while (content.match(r)) {
+		content = content.replace(r, "")
+	}
+	// Create a sprite sheet of known and unknown emojis from the end of the message
+	const buffer = await emojiSheet.compositeMatrixEmojis(endOfMessageEmojis, mxcDownloader)
+	// Attach it
+	const name = "emojis.png"
+	attachments.push({id: String(attachments.length), name})
+	pendingFiles.push({name, buffer})
+	return content
+}
+
+/**
+ * @param {string} input
+ * @param {{api: import("../../matrix/api")}} di simple-as-nails dependency injection for the matrix API
+ */
+async function handleRoomOrMessageLinks(input, di) {
+	let offset = 0
+	for (const match of [...input.matchAll(/("?https:\/\/matrix.to\/#\/(![^"/, ?)]+)(?:\/(\$[^"/ ?)]+))?(?:\?[^",:!? )]*?)?)(">|[,<\n )]|$)/g)]) {
+		assert(typeof match.index === "number")
+		const [_, attributeValue, roomID, eventID, endMarker] = match
+		let result
+
+		const resultType = endMarker === '">' ? "html" : "plain"
+		const MAKE_RESULT = {
+			ROOM_LINK: {
+				html: channelID => `${attributeValue}" data-channel-id="${channelID}">`,
+				plain: channelID => `<#${channelID}>${endMarker}`
+			},
+			MESSAGE_LINK: {
+				html: (guildID, channelID, messageID) => `${attributeValue}" data-channel-id="${channelID}" data-guild-id="${guildID}" data-message-id="${messageID}">`,
+				plain: (guildID, channelID, messageID) => `https://discord.com/channels/${guildID}/${channelID}/${messageID}${endMarker}`
+			}
+		}
+
+		// Don't process links that are part of the reply fallback, they'll be removed entirely by turndown
+		if (input.slice(match.index + match[0].length + offset).startsWith("In reply to")) continue
+
+		const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
+		if (!channelID) continue
+		if (!eventID) {
+			// 1: It's a room link, so <#link> to the channel
+			result = MAKE_RESULT.ROOM_LINK[resultType](channelID)
+		} else {
+			// Linking to a particular event with a discord.com/channels/guildID/channelID/messageID link
+			// Need to know the guildID and messageID
+			const guildID = discord.channels.get(channelID)?.["guild_id"]
+			if (!guildID) continue
+			const messageID = select("event_message", "message_id", {event_id: eventID}).pluck().get()
+			if (messageID) {
+				// 2: Linking to a known event
+				result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, messageID)
+			} else {
+				// 3: Linking to an unknown event that OOYE didn't originally bridge - we can guess messageID from the timestamp
+				let originalEvent
+				try {
+					originalEvent = await di.api.getEvent(roomID, eventID)
+				} catch (e) {
+					continue // Our homeserver doesn't know about the event, so can't resolve it to a Discord link
+				}
+				const guessedMessageID = dUtils.timestampToSnowflakeInexact(originalEvent.origin_server_ts)
+				result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, guessedMessageID)
+			}
+		}
+
+		input = input.slice(0, match.index + offset) + result + input.slice(match.index + match[0].length + offset)
+		offset += result.length - match[0].length
+	}
+
+	return input
+}
+
+/**
+ * @param {string} content
+ * @param {string} senderMxid
+ * @param {string} roomID
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"]}} di
+ */
+async function checkWrittenMentions(content, senderMxid, roomID, guild, di) {
+	let writtenMentionMatch = content.match(/(?:^|[^"[<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // /d flag for indices requires node.js 16+
+	if (writtenMentionMatch) {
+		if (writtenMentionMatch[1] === "room") { // convert @room to @everyone
+			const powerLevels = await di.api.getStateEvent(roomID, "m.room.power_levels", "")
+			const userPower = powerLevels.users?.[senderMxid] || 0
+			if (userPower >= powerLevels.notifications?.room) {
+				return {
+					// @ts-ignore - typescript doesn't know about indices yet
+					content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `@everyone` + content.slice(writtenMentionMatch.indices[1][1]),
+					ensureJoined: [],
+					allowedMentionsParse: ["everyone"]
+				}
+			}
+		} else {
+			const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]})
+			if (results[0]) {
+				assert(results[0].user)
+				return {
+					// @ts-ignore - typescript doesn't know about indices yet
+					content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `<@${results[0].user.id}>` + content.slice(writtenMentionMatch.indices[1][1]),
+					ensureJoined: [results[0].user],
+					allowedMentionsParse: []
+				}
+			}
+		}
+	}
+}
+
+/**
+ * @param {Element} node
+ * @param {string[]} tagNames allcaps tag names
+ * @returns {any | undefined} the node you were checking for, or undefined
+ */
+function nodeIsChildOf(node, tagNames) {
+	// @ts-ignore
+	for (; node; node = node.parentNode) if (tagNames.includes(node.tagName)) return node
+}
+
+const attachmentEmojis = new Map([
+	["m.image", "🖼️"],
+	["m.video", "🎞️"],
+	["m.audio", "🎶"],
+	["m.file", "📄"]
+])
+
+/**
+ * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event
+ * @param {import("discord-api-types/v10").APIGuild} guild
+ * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"], mxcDownloader: (mxc: string) => Promise<Buffer | undefined>}} di simple-as-nails dependency injection for the matrix API
+ */
+async function eventToMessage(event, guild, di) {
+	let displayName = event.sender
+	let avatarURL = undefined
+	const allowedMentionsParse = ["users", "roles"]
+	/** @type {string[]} */
+	let messageIDsToEdit = []
+	let replyLine = ""
+	// Extract a basic display name from the sender
+	const match = event.sender.match(/^@(.*?):/)
+	if (match) displayName = match[1]
+	// Try to extract an accurate display name and avatar URL from the member event
+	const member = await getMemberFromCacheOrHomeserver(event.room_id, event.sender, di?.api)
+	if (member.displayname) displayName = member.displayname
+	if (member.avatar_url) avatarURL = mxUtils.getPublicUrlForMxc(member.avatar_url) || undefined
+	// If the display name is too long to be put into the webhook (80 characters is the maximum),
+	// put the excess characters into displayNameRunoff, later to be put at the top of the message
+	let [displayNameShortened, displayNameRunoff] = splitDisplayName(displayName)
+	// If the message type is m.emote, the full name is already included at the start of the message, so remove any runoff
+	if (event.type === "m.room.message" && event.content.msgtype === "m.emote") {
+		displayNameRunoff = ""
+	}
+
+	let content = event.content.body // ultimate fallback
+	const attachments = []
+	/** @type {({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer})[]} */
+	const pendingFiles = []
+	/** @type {DiscordTypes.APIUser[]} */
+	const ensureJoined = []
+
+	// Convert content depending on what the message is
+	if (event.type === "m.room.message" && (event.content.msgtype === "m.text" || event.content.msgtype === "m.emote")) {
+		// Handling edits. If the edit was an edit of a reply, edits do not include the reply reference, so we need to fetch up to 2 more events.
+		// this event ---is an edit of--> original event ---is a reply to--> past event
+		await (async () => {
+			// Check if there is an edit
+			const relatesTo = event.content["m.relates_to"]
+			if (!event.content["m.new_content"] || !relatesTo || relatesTo.rel_type !== "m.replace") return
+			// Check if we have a pointer to what was edited
+			const originalEventId = relatesTo.event_id
+			if (!originalEventId) return
+			messageIDsToEdit = select("event_message", "message_id", {event_id: originalEventId}, "ORDER BY part").pluck().all()
+			if (!messageIDsToEdit.length) return
+
+			// Ok, it's an edit.
+			event.content = event.content["m.new_content"]
+
+			// Is it editing a reply? We need special handling if it is.
+			// Get the original event, then check if it was a reply
+			const originalEvent = await di.api.getEvent(event.room_id, originalEventId)
+			if (!originalEvent) return
+			const repliedToEventId = originalEvent.content["m.relates_to"]?.["m.in_reply_to"]?.event_id
+			if (!repliedToEventId) return
+
+			// After all that, it's an edit of a reply.
+			// We'll be sneaky and prepare the message data so that the next steps can handle it just like original messages.
+			Object.assign(event.content, {
+				"m.relates_to": {
+					"m.in_reply_to": {
+						event_id: repliedToEventId
+					}
+				}
+			})
+		})()
+
+		// Handling replies. We'll look up the data of the replied-to event from the Matrix homeserver.
+		// Note that an <mx-reply> element is not guaranteed because this might be m.new_content.
+		await (async () => {
+			const repliedToEventId = event.content["m.relates_to"]?.["m.in_reply_to"]?.event_id
+			if (!repliedToEventId) return
+			let repliedToEvent
+			try {
+				repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId)
+			} catch (e) {
+				// Original event isn't on our homeserver, so we'll *partially* trust the client's reply fallback.
+				// We'll trust the fallback's quoted content and put it in the reply preview, but we won't trust the authorship info on it.
+
+				// (But if the fallback's quoted content doesn't exist, we give up. There's nothing for us to quote.)
+				if (event.content["format"] !== "org.matrix.custom.html" || typeof event.content["formatted_body"] !== "string") {
+					const lines = event.content.body.split("\n")
+					let stage = 0
+					for (let i = 0; i < lines.length; i++) {
+						if (stage >= 0 && lines[i][0] === ">") stage = 1
+						if (stage >= 1 && lines[i].trim() === "") stage = 2
+						if (stage === 2 && lines[i].trim() !== "") {
+							event.content.body = lines.slice(i).join("\n")
+							break
+						}
+					}
+					return
+				}
+				const mxReply = event.content["formatted_body"]
+				const quoted = mxReply.match(/^<mx-reply><blockquote>.*?In reply to.*?<br>(.*)<\/blockquote><\/mx-reply>/)?.[1]
+				if (!quoted) return
+				const contentPreviewChunks = chunk(
+					entities.decodeHTML5Strict( // Remove entities like &amp; &quot;
+						quoted.replace(/^\s*<blockquote>.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards
+							.replace(/(?:\n|<br>)+/g, " ") // Should all be on one line
+							.replace(/<span [^>]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
+							.replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting.
+					), 50)
+				replyLine = "-# > " + contentPreviewChunks[0]
+				if (contentPreviewChunks.length > 1) replyLine = replyLine.replace(/[,.']$/, "") + "..."
+				replyLine += "\n"
+				return
+			}
+
+			// @ts-ignore
+			const autoEmoji = new Map(select("auto_emoji", ["name", "emoji_id"], {}, "WHERE name = 'L1' OR name = 'L2'").raw().all())
+			replyLine = `<:L1:${autoEmoji.get("L1")}><:L2:${autoEmoji.get("L2")}>`
+			const row = from("event_message").join("message_channel", "message_id").select("channel_id", "message_id").where({event_id: repliedToEventId}).and("ORDER BY part").get()
+			if (row) {
+				replyLine += `https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} `
+			}
+			const sender = repliedToEvent.sender
+			const authorID = getUserOrProxyOwnerID(sender)
+			if (authorID) {
+				replyLine += `<@${authorID}>`
+			} else {
+				let senderName = select("member_cache", "displayname", {mxid: sender}).pluck().get()
+				if (!senderName) {
+					const match = sender.match(/@([^:]*)/)
+					assert(match)
+					senderName = match[1]
+				}
+				replyLine += `**Ⓜ${senderName}**`
+			}
+			// If the event has been edited, the homeserver will include the relation in `unsigned`.
+			if (repliedToEvent.unsigned?.["m.relations"]?.["m.replace"]?.content?.["m.new_content"]) {
+				repliedToEvent = repliedToEvent.unsigned["m.relations"]["m.replace"] // Note: this changes which event_id is in repliedToEvent.
+				repliedToEvent.content = repliedToEvent.content["m.new_content"]
+			}
+			let contentPreview
+			const fileReplyContentAlternative = attachmentEmojis.get(repliedToEvent.content.msgtype)
+			if (fileReplyContentAlternative) {
+				contentPreview = " " + fileReplyContentAlternative
+			} else if (repliedToEvent.unsigned?.redacted_because) {
+				contentPreview = " (in reply to a deleted message)"
+			} else {
+				// Generate a reply preview for a standard message
+				/** @type {string} */
+				let repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body
+				repliedToContent = repliedToContent.replace(/.*<\/mx-reply>/s, "") // Remove everything before replies, so just use the actual message body
+				repliedToContent = repliedToContent.replace(/^\s*<blockquote>.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards
+				repliedToContent = repliedToContent.replace(/(?:\n|<br>)+/g, " ") // Should all be on one line
+				repliedToContent = repliedToContent.replace(/<span [^>]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
+				repliedToContent = repliedToContent.replace(/<img([^>]*)>/g, (_, att) => { // Convert Matrix emoji images into Discord emoji markdown
+					const mxcUrlMatch = att.match(/\bsrc="(mxc:\/\/[^"]+)"/)
+					const titleTextMatch = att.match(/\btitle=":?([^:"]+)/)
+					return convertEmoji(mxcUrlMatch?.[1], titleTextMatch?.[1], false, false)
+				})
+				repliedToContent = repliedToContent.replace(/<[^:>][^>]*>/g, "") // Completely strip all HTML tags and formatting.
+				repliedToContent = repliedToContent.replace(/\bhttps?:\/\/[^ )]*/g, "<$&>")
+				repliedToContent = entities.decodeHTML5Strict(repliedToContent) // Remove entities like &amp; &quot;
+				const contentPreviewChunks = chunk(repliedToContent, 50)
+				if (contentPreviewChunks.length) {
+					contentPreview = ": " + contentPreviewChunks[0]
+					if (contentPreviewChunks.length > 1) contentPreview = contentPreview.replace(/[,.']$/, "") + "..."
+				} else {
+					console.log("Unable to generate reply preview for this replied-to event because we stripped all of it:", repliedToEvent)
+					contentPreview = ""
+				}
+			}
+			replyLine = `-# > ${replyLine}${contentPreview}\n`
+		})()
+
+		if (event.content.format === "org.matrix.custom.html" && event.content.formatted_body) {
+			let input = event.content.formatted_body
+			if (event.content.msgtype === "m.emote") {
+				input = `* ${displayName} ${input}`
+			}
+
+			// Handling mentions of Discord users
+			input = input.replace(/("https:\/\/matrix.to\/#\/((?:@|%40)[^"]+)")>/g, (whole, attributeValue, mxid) => {
+				mxid = decodeURIComponent(mxid)
+				if (mxUtils.eventSenderIsFromDiscord(mxid)) {
+					// Handle mention of an OOYE sim user by their mxid
+					const id = select("sim", "user_id", {mxid}).pluck().get()
+					if (!id) return whole
+					return `${attributeValue} data-user-id="${id}">`
+				} else {
+					// Handle mention of a Matrix user by their mxid
+					// Check if this Matrix user is actually the sim user from another old bridge in the room?
+					const match = mxid.match(/[^:]*discord[^:]*_([0-9]{6,}):/) // try to match @_discord_123456, @_discordpuppet_123456, etc.
+					if (match) return `${attributeValue} data-user-id="${match[1]}">`
+					// Nope, just a real Matrix user.
+					return whole
+				}
+			})
+
+			// Handling mentions of rooms and room-messages
+			input = await handleRoomOrMessageLinks(input, di)
+
+			// Stripping colons after mentions
+			input = input.replace(/( data-user-id.*?<\/a>):?/g, "$1")
+			input = input.replace(/("https:\/\/matrix.to.*?<\/a>):?/g, "$1")
+
+			// Element adds a bunch of <br> before </blockquote> but doesn't render them. I can't figure out how this even works in the browser, so let's just delete those.
+			input = input.replace(/(?:\n|<br ?\/?>\s*)*<\/blockquote>/g, "</blockquote>")
+
+			// The matrix spec hasn't decided whether \n counts as a newline or not, but I'm going to count it, because if it's in the data it's there for a reason.
+			// But I should not count it if it's between block elements.
+			input = input.replace(/(<\/?([^ >]+)[^>]*>)?\n(<\/?([^ >]+)[^>]*>)?/g, (whole, beforeContext, beforeTag, afterContext, afterTag) => {
+				// console.error(beforeContext, beforeTag, afterContext, afterTag)
+				if (typeof beforeTag !== "string" && typeof afterTag !== "string") {
+					return "<br>"
+				}
+				beforeContext = beforeContext || ""
+				beforeTag = beforeTag || ""
+				afterContext = afterContext || ""
+				afterTag = afterTag || ""
+				if (!mxUtils.BLOCK_ELEMENTS.includes(beforeTag.toUpperCase()) && !mxUtils.BLOCK_ELEMENTS.includes(afterTag.toUpperCase())) {
+					return beforeContext + "<br>" + afterContext
+				} else {
+					return whole
+				}
+			})
+
+			// Note: Element's renderers on Web and Android currently collapse whitespace, like the browser does. Turndown also collapses whitespace which is good for me.
+			// If later I'm using a client that doesn't collapse whitespace and I want turndown to follow suit, uncomment the following line of code, and it Just Works:
+			// input = input.replace(/ /g, "&nbsp;")
+			// There is also a corresponding test to uncomment, named "event2message: whitespace is retained"
+
+			// Handling written @mentions: we need to look for candidate Discord members to join to the room
+			// This shouldn't apply to code blocks, links, or inside attributes. So editing the HTML tree instead of regular expressions is a sensible choice here.
+			// We're using the domino parser because Turndown uses the same and can reuse this tree.
+			const doc = domino.createDocument(
+				// DOM parsers arrange elements in the <head> and <body>. Wrapping in a custom element ensures elements are reliably arranged in a single element.
+				'<x-turndown id="turndown-root">' + input + '</x-turndown>'
+			);
+			const root = doc.getElementById("turndown-root");
+			async function forEachNode(node) {
+				for (; node; node = node.nextSibling) {
+					// Check written mentions
+					if (node.nodeType === 3 && node.nodeValue.includes("@") && !nodeIsChildOf(node, ["A", "CODE", "PRE"])) {
+						const result = await checkWrittenMentions(node.nodeValue, event.sender, event.room_id, guild, di)
+						if (result) {
+							node.nodeValue = result.content
+							ensureJoined.push(...result.ensureJoined)
+							allowedMentionsParse.push(...result.allowedMentionsParse)
+						}
+					}
+					// Check for incompatible backticks in code blocks
+					let preNode
+					if (node.nodeType === 3 && node.nodeValue.includes("```") && (preNode = nodeIsChildOf(node, ["PRE"]))) {
+						if (preNode.firstChild?.nodeName === "CODE") {
+							const ext = preNode.firstChild.className.match(/language-(\S+)/)?.[1] || "txt"
+							const filename = `inline_code.${ext}`
+							// Build the replacement <code> node
+							const replacementCode = doc.createElement("code")
+							replacementCode.textContent = `[${filename}]`
+							// Build its containing <span> node
+							const replacement = doc.createElement("span")
+							replacement.appendChild(doc.createTextNode(" "))
+							replacement.appendChild(replacementCode)
+							replacement.appendChild(doc.createTextNode(" "))
+							// Replace the code block with the <span>
+							preNode.replaceWith(replacement)
+							// Upload the code as an attachment
+							const content = getCodeContent(preNode.firstChild)
+							attachments.push({id: String(attachments.length), filename})
+							pendingFiles.push({name: filename, buffer: Buffer.from(content, "utf8")})
+						}
+					}
+					await forEachNode(node.firstChild)
+				}
+			}
+			await forEachNode(root)
+
+			// SPRITE SHEET EMOJIS FEATURE: Emojis at the end of the message that we don't know about will be reuploaded as a sprite sheet.
+			// First we need to determine which emojis are at the end.
+			endOfMessageEmojis = []
+			let match
+			let last = input.length
+			while ((match = input.slice(0, last).match(/<img [^>]*>\s*$/))) {
+				if (!match[0].includes("data-mx-emoticon")) break
+				const mxcUrl = match[0].match(/\bsrc="(mxc:\/\/[^"]+)"/)
+				if (mxcUrl) endOfMessageEmojis.unshift(mxcUrl[1])
+				assert(typeof match.index === "number", "Your JavaScript implementation does not comply with TC39: https://tc39.es/ecma262/multipage/text-processing.html#sec-regexpbuiltinexec")
+				last = match.index
+			}
+
+			// @ts-ignore bad type from turndown
+			content = turndownService.turndown(root)
+
+			// Put < > around any surviving matrix.to links to hide the URL previews
+			content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/g, "<$&>")
+
+			// It's designed for commonmark, we need to replace the space-space-newline with just newline
+			content = content.replace(/  \n/g, "\n")
+
+			// If there's a blockquote at the start of the message body and this message is a reply, they should be visually separated
+			if (replyLine && content.startsWith("> ")) content = "\n" + content
+
+			// SPRITE SHEET EMOJIS FEATURE:
+			content = await uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles, di?.mxcDownloader)
+		} else {
+			// Looks like we're using the plaintext body!
+			content = event.content.body
+
+			if (event.content.msgtype === "m.emote") {
+				content = `* ${displayName} ${content}`
+			}
+
+			content = await handleRoomOrMessageLinks(content, di) // Replace matrix.to links with discord.com equivalents where possible
+			content = content.replace(/\bhttps?:\/\/matrix\.to\/[^<>\n )]*/, "<$&>") // Put < > around any surviving matrix.to links to hide the URL previews
+
+			const result = await checkWrittenMentions(content, event.sender, event.room_id, guild, di)
+			if (result) {
+				content = result.content
+				ensureJoined.push(...result.ensureJoined)
+				allowedMentionsParse.push(...result.allowedMentionsParse)
+			}
+
+			// Markdown needs to be escaped, though take care not to escape the middle of links
+			// @ts-ignore bad type from turndown
+			content = turndownService.escape(content)
+		}
+	} else if (event.type === "m.room.message" && (event.content.msgtype === "m.file" || event.content.msgtype === "m.video" || event.content.msgtype === "m.audio" || event.content.msgtype === "m.image")) {
+		content = ""
+		const filename = event.content.filename || event.content.body
+		// A written `event.content.body` will be bridged to Discord's image `description` which is like alt text.
+		// Bridging as description rather than message content in order to match Matrix clients (Element, Neochat) which treat this as alt text or title text.
+		const description = (event.content.body !== event.content.filename && event.content.filename && event.content.body) || undefined
+		if ("url" in event.content) {
+			// Unencrypted
+			const url = mxUtils.getPublicUrlForMxc(event.content.url)
+			assert(url)
+			attachments.push({id: "0", description, filename})
+			pendingFiles.push({name: filename, url})
+		} else {
+			// Encrypted
+			const url = mxUtils.getPublicUrlForMxc(event.content.file.url)
+			assert(url)
+			assert.equal(event.content.file.key.alg, "A256CTR")
+			attachments.push({id: "0", description, filename})
+			pendingFiles.push({name: filename, url, key: event.content.file.key.k, iv: event.content.file.iv})
+		}
+	} else if (event.type === "m.sticker") {
+		content = ""
+		const url = mxUtils.getPublicUrlForMxc(event.content.url)
+		assert(url)
+		let filename = event.content.body
+		if (event.type === "m.sticker") {
+			let mimetype
+			if (event.content.info?.mimetype?.includes("/")) {
+				mimetype = event.content.info.mimetype
+			} else {
+				const res = await di.fetch(url, {method: "HEAD"})
+				if (res.status === 200) {
+					mimetype = res.headers.get("content-type")
+				}
+				if (!mimetype) throw new Error(`Server error ${res.status} or missing content-type while detecting sticker mimetype`)
+			}
+			filename += "." + mimetype.split("/")[1]
+		}
+		attachments.push({id: "0", filename})
+		pendingFiles.push({name: filename, url})
+	}
+
+	content = displayNameRunoff + replyLine + content
+
+	// Split into 2000 character chunks
+	const chunks = chunk(content, 2000)
+	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */
+	const messages = chunks.map(content => ({
+		content,
+		allowed_mentions: {
+			parse: allowedMentionsParse
+		},
+		username: displayNameShortened,
+		avatar_url: avatarURL
+	}))
+
+	if (attachments.length) {
+		// If content is empty (should be the case when uploading a file) then chunk-text will create 0 messages.
+		// There needs to be a message to add attachments to.
+		if (!messages.length) messages.push({
+			content,
+			username: displayNameShortened,
+			avatar_url: avatarURL
+		})
+		messages[0].attachments = attachments
+		// @ts-ignore these will be converted to real files when the message is about to be sent
+		messages[0].pendingFiles = pendingFiles
+	}
+
+	const messagesToEdit = []
+	const messagesToSend = []
+	for (let i = 0; i < messages.length; i++) {
+		const next = messageIDsToEdit[0]
+		if (next) {
+			messagesToEdit.push({id: next, message: messages[i]})
+			messageIDsToEdit.shift()
+		} else {
+			messagesToSend.push(messages[i])
+		}
+	}
+
+	// Ensure there is code coverage for adding, editing, and deleting
+	if (messagesToSend.length) void 0
+	if (messagesToEdit.length) void 0
+	if (messageIDsToEdit.length) void 0
+
+	return {
+		messagesToEdit,
+		messagesToSend,
+		messagesToDelete: messageIDsToEdit,
+		ensureJoined
+	}
+}
+
+module.exports.eventToMessage = eventToMessage

src/m2d/converters/utils.js

@@ -0,0 +1,243 @@
+// @ts-check
+
+const assert = require("assert").strict
+
+const passthrough = require("../../passthrough")
+const {db} = passthrough
+
+const {reg} = require("../../matrix/read-registration")
+const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
+
+/** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore
+let hasher = null
+// @ts-ignore
+require("xxhash-wasm")().then(h => hasher = h)
+
+const BLOCK_ELEMENTS = [
+	"ADDRESS", "ARTICLE", "ASIDE", "AUDIO", "BLOCKQUOTE", "BODY", "CANVAS",
+	"CENTER", "DD", "DETAILS", "DIR", "DIV", "DL", "DT", "FIELDSET", "FIGCAPTION", "FIGURE",
+	"FOOTER", "FORM", "FRAMESET", "H1", "H2", "H3", "H4", "H5", "H6", "HEADER",
+	"HGROUP", "HR", "HTML", "ISINDEX", "LI", "MAIN", "MENU", "NAV", "NOFRAMES",
+	"NOSCRIPT", "OL", "OUTPUT", "P", "PRE", "SECTION", "SUMMARY", "TABLE", "TBODY", "TD",
+	"TFOOT", "TH", "THEAD", "TR", "UL"
+]
+const NEWLINE_ELEMENTS = BLOCK_ELEMENTS.concat(["BR"])
+
+/**
+ * Determine whether an event is the bridged representation of a discord message.
+ * Such messages shouldn't be bridged again.
+ * @param {string} sender
+ */
+function eventSenderIsFromDiscord(sender) {
+	// If it's from a user in the bridge's namespace, then it originated from discord
+	// This includes messages sent by the appservice's bot user, because that is what's used for webhooks
+	// TODO: It would be nice if bridge system messages wouldn't trigger this check and could be bridged from matrix to discord, while webhook reflections would remain ignored...
+	// TODO that only applies to the above todo: But you'd have to watch out for the /icon command, where the bridge bot would set the room avatar, and that shouldn't be reflected into the room a second time.
+	if (userRegex.some(x => sender.match(x))) {
+		return true
+	}
+
+	return false
+}
+
+/**
+ * Event IDs are really big and have more entropy than we need.
+ * If we want to store the event ID in the database, we can store a more compact version by hashing it with this.
+ * I choose a 64-bit non-cryptographic hash as only a 32-bit hash will see birthday collisions unreasonably frequently: https://en.wikipedia.org/wiki/Birthday_attack#Mathematics
+ * xxhash outputs an unsigned 64-bit integer.
+ * Converting to a signed 64-bit integer with no bit loss so that it can be stored in an SQLite integer field as-is: https://www.sqlite.org/fileformat2.html#record_format
+ * This should give very efficient storage with sufficient entropy.
+ * @param {string} eventID
+ */
+function getEventIDHash(eventID) {
+	assert(hasher, "xxhash is not ready yet")
+	if (eventID[0] === "$" && eventID.length >= 13) {
+		eventID = eventID.slice(1) // increase entropy per character to potentially help xxhash
+	}
+	const unsignedHash = hasher.h64(eventID)
+	const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
+	return signedHash
+}
+
+class MatrixStringBuilder {
+	constructor() {
+		this.body = ""
+		this.formattedBody = ""
+	}
+
+	/**
+	 * @param {string} body
+	 * @param {string} [formattedBody]
+	 * @param {any} [condition]
+	 */
+	add(body, formattedBody, condition = true) {
+		if (condition) {
+			if (formattedBody == undefined) formattedBody = body
+			this.body += body
+			this.formattedBody += formattedBody
+		}
+		return this
+	}
+
+	/**
+	 * @param {string} body
+	 * @param {string} [formattedBody]
+	 * @param {any} [condition]
+	 */
+	addLine(body, formattedBody, condition = true) {
+		if (condition) {
+			if (formattedBody == undefined) formattedBody = body
+			if (this.body.length && this.body.slice(-1) !== "\n") this.body += "\n"
+			this.body += body
+			const match = this.formattedBody.match(/<\/?([a-zA-Z]+[a-zA-Z0-9]*)[^>]*>\s*$/)
+			if (this.formattedBody.length && (!match || !NEWLINE_ELEMENTS.includes(match[1].toUpperCase()))) this.formattedBody += "<br>"
+			this.formattedBody += formattedBody
+		}
+		return this
+	}
+
+	/**
+	 * @param {string} body
+	 * @param {string} [formattedBody]
+	 * @param {any} [condition]
+	 */
+	addParagraph(body, formattedBody, condition = true) {
+		if (condition) {
+			if (formattedBody == undefined) formattedBody = body
+			if (this.body.length && this.body.slice(-1) !== "\n") this.body += "\n\n"
+			this.body += body
+			formattedBody = `<p>${formattedBody}</p>`
+			this.formattedBody += formattedBody
+		}
+		return this
+	}
+
+	get() {
+		return {
+			msgtype: "m.text",
+			body: this.body,
+			format: "org.matrix.custom.html",
+			formatted_body: this.formattedBody
+		}
+	}
+}
+
+/**
+ * Context: Room IDs are not routable on their own. Room permalinks need a list of servers to try. The client is responsible for coming up with a list of servers.
+ * ASSUMPTION 1: The bridge bot is a member of the target room and can therefore access its power levels and member list for calculation.
+ * ASSUMPTION 2: Because the bridge bot is a member of the target room, the target room is bridged.
+ * https://spec.matrix.org/v1.9/appendices/#routing
+ * https://gitdab.com/cadence/out-of-your-element/issues/11
+ * @param {string} roomID
+ * @param {{[K in "getStateEvent" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api
+ */
+async function getViaServers(roomID, api) {
+	const candidates = []
+	const {joined} = await api.getJoinedMembers(roomID)
+	// Candidate 0: The bot's own server name
+	candidates.push(reg.ooye.server_name)
+	// Candidate 1: Highest joined non-sim non-bot power level user in the room
+	// https://github.com/matrix-org/matrix-react-sdk/blob/552c65db98b59406fb49562e537a2721c8505517/src/utils/permalinks/Permalinks.ts#L172
+	try {
+		/** @type {{users?: {[mxid: string]: number}}} */
+		const powerLevels = await api.getStateEvent(roomID, "m.room.power_levels", "")
+		if (powerLevels.users) {
+			const sorted = Object.entries(powerLevels.users).sort((a, b) => b[1] - a[1]) // Highest...
+			for (const power of sorted) {
+				const mxid = power[0]
+				if (!(mxid in joined)) continue // joined...
+				if (userRegex.some(r => mxid.match(r))) continue // non-sim non-bot...
+				const match = mxid.match(/:(.*)/)
+				assert(match)
+				if (!candidates.includes(match[1])) {
+					candidates.push(match[1])
+					break
+				}
+			}
+		}
+	} catch (e) {
+		// power levels event not found
+	}
+	// Candidates 2-3: Most popular servers in the room
+	/** @type {Map<string, number>} */
+	const servers = new Map()
+	// We can get the most popular servers if we know the members, so let's process those...
+	Object.keys(joined)
+		.filter(mxid => !mxid.startsWith("@_")) // Quick check
+		.filter(mxid => !userRegex.some(r => mxid.match(r))) // Full check
+		.slice(0, 1000) // Just sample the first thousand real members
+		.map(mxid => {
+			const match = mxid.match(/:(.*)/)
+			assert(match)
+			return match[1]
+		})
+		.filter(server => !server.match(/([a-f0-9:]+:+)+[a-f0-9]+/)) // No IPv6 servers
+		.filter(server => !server.match(/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)) // No IPv4 servers
+		// I don't care enough to check ACLs
+		.forEach(server => {
+			const existing = servers.get(server)
+			if (!existing) servers.set(server, 1)
+			else servers.set(server, existing + 1)
+		})
+	const serverList = [...servers.entries()].sort((a, b) => b[1] - a[1])
+	for (const server of serverList) {
+		if (!candidates.includes(server[0])) {
+			candidates.push(server[0])
+			if (candidates.length >= 4) break // Can have at most 4 candidate via servers
+		}
+	}
+	return candidates
+}
+
+/**
+ * Context: Room IDs are not routable on their own. Room permalinks need a list of servers to try. The client is responsible for coming up with a list of servers.
+ * ASSUMPTION 1: The bridge bot is a member of the target room and can therefore access its power levels and member list for calculation.
+ * ASSUMPTION 2: Because the bridge bot is a member of the target room, the target room is bridged.
+ * https://spec.matrix.org/v1.9/appendices/#routing
+ * https://gitdab.com/cadence/out-of-your-element/issues/11
+ * @param {string} roomID
+ * @param {{[K in "getStateEvent" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api
+ * @returns {Promise<URLSearchParams>}
+ */
+async function getViaServersQuery(roomID, api) {
+	const list = await getViaServers(roomID, api)
+	const qs = new URLSearchParams()
+	for (const server of list) {
+		qs.append("via", server)
+	}
+	return qs
+}
+
+/**
+ * Since the introduction of authenticated media, this can no longer just be the /_matrix/media/r0/download URL
+ * because Discord and Discord users cannot use those URLs. Media now has to be proxied through the bridge.
+ * To avoid the bridge acting as a proxy for *any* media, there is a list of permitted media stored in the database.
+ * (The other approach would be signing the URLs with a MAC (or similar) and adding the signature, but I'm not a
+ * cryptographer, so I don't want to.) To reduce database disk space usage, instead of storing each permitted URL,
+ * we just store its xxhash as a signed (as in +/-, not signature) 64-bit integer, which fits in an SQLite integer field.
+ * @see https://matrix.org/blog/2024/06/26/sunsetting-unauthenticated-media/ background
+ * @see https://matrix.org/blog/2024/06/20/matrix-v1.11-release/ implementation details
+ * @see https://www.sqlite.org/fileformat2.html#record_format SQLite integer field size
+ * @param {string} mxc
+ * @returns {string?}
+ */
+function getPublicUrlForMxc(mxc) {
+	assert(hasher, "xxhash is not ready yet")
+	const avatarURLParts = mxc?.match(/^mxc:\/\/([^/]+)\/(\w+)$/)
+	if (!avatarURLParts) return null
+
+	const serverAndMediaID = `${avatarURLParts[1]}/${avatarURLParts[2]}`
+	const unsignedHash = hasher.h64(serverAndMediaID)
+	const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
+	db.prepare("INSERT OR IGNORE INTO media_proxy (permitted_hash) VALUES (?)").run(signedHash)
+
+	return `${reg.ooye.bridge_origin}/download/matrix/${serverAndMediaID}`
+}
+
+module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS
+module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord
+module.exports.getPublicUrlForMxc = getPublicUrlForMxc
+module.exports.getEventIDHash = getEventIDHash
+module.exports.MatrixStringBuilder = MatrixStringBuilder
+module.exports.getViaServers = getViaServers
+module.exports.getViaServersQuery = getViaServersQuery

src/m2d/converters/utils.test.js

@@ -0,0 +1,178 @@
+// @ts-check
+
+const e = new Error("Custom error")
+
+const {test} = require("supertape")
+const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder, getViaServers} = require("./utils")
+const util = require("util")
+
+/** @param {string[]} mxids */
+function joinedList(mxids) {
+	/** @type {{[mxid: string]: {display_name: null, avatar_url: null}}} */
+	const joined = {}
+	for (const mxid of mxids) {
+		joined[mxid] = {
+			display_name: null,
+			avatar_url: null
+		}
+	}
+	return {joined}
+}
+
+test("sender type: matrix user", t => {
+	t.notOk(eventSenderIsFromDiscord("@cadence:cadence.moe"))
+})
+
+test("sender type: ooye bot", t => {
+	t.ok(eventSenderIsFromDiscord("@_ooye_bot:cadence.moe"))
+})
+
+test("sender type: ooye puppet", t => {
+	t.ok(eventSenderIsFromDiscord("@_ooye_sheep:cadence.moe"))
+})
+
+test("event hash: hash is the same each time", t => {
+	const eventID = "$example"
+	t.equal(getEventIDHash(eventID), getEventIDHash(eventID))
+})
+
+test("event hash: hash is different for different inputs", t => {
+	t.notEqual(getEventIDHash("$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe1"), getEventIDHash("$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe2"))
+})
+
+test("MatrixStringBuilder: add, addLine, add same text", t => {
+	const gatewayMessage = {t: "MY_MESSAGE", d: {display: "Custom message data"}}
+	let stackLines = e.stack?.split("\n")
+
+	const builder = new MatrixStringBuilder()
+	builder.addLine("\u26a0 Bridged event from Discord not delivered", "\u26a0 <strong>Bridged event from Discord not delivered</strong>")
+	builder.addLine(`Gateway event: ${gatewayMessage.t}`)
+	builder.addLine(e.toString())
+	if (stackLines) {
+		stackLines = stackLines.slice(0, 2)
+		stackLines[1] = stackLines[1].replace(/\\/g, "/").replace(/(\s*at ).*(\/m2d\/)/, "$1.$2")
+		builder.addLine(`Error trace:`, `<details><summary>Error trace</summary>`)
+		builder.add(`\n${stackLines.join("\n")}`, `<pre>${stackLines.join("\n")}</pre></details>`)
+	}
+	builder.addLine("", `<details><summary>Original payload</summary><pre>${util.inspect(gatewayMessage.d, false, 4, false)}</pre></details>`)
+
+	t.deepEqual(builder.get(), {
+		msgtype: "m.text",
+		body: "\u26a0 Bridged event from Discord not delivered"
+			+ "\nGateway event: MY_MESSAGE"
+			+ "\nError: Custom error"
+			+ "\nError trace:"
+			+ "\nError: Custom error"
+			+ "\n    at ./m2d/converters/utils.test.js:3:11)\n",
+		format: "org.matrix.custom.html",
+		formatted_body: "\u26a0 <strong>Bridged event from Discord not delivered</strong>"
+			+ "<br>Gateway event: MY_MESSAGE"
+			+ "<br>Error: Custom error"
+			+ "<br><details><summary>Error trace</summary><pre>Error: Custom error\n    at ./m2d/converters/utils.test.js:3:11)</pre></details>"
+			+ `<details><summary>Original payload</summary><pre>{ display: 'Custom message data' }</pre></details>`
+	})
+})
+
+test("MatrixStringBuilder: complete code coverage", t => {
+	const builder = new MatrixStringBuilder()
+	builder.add("Line 1")
+	builder.addParagraph("Line 2")
+	builder.add("Line 3")
+	builder.addParagraph("Line 4")
+
+	t.deepEqual(builder.get(), {
+		msgtype: "m.text",
+		body: "Line 1\n\nLine 2Line 3\n\nLine 4",
+		format: "org.matrix.custom.html",
+		formatted_body: "Line 1<p>Line 2</p>Line 3<p>Line 4</p>"
+	})
+})
+
+test("getViaServers: returns the server name if the room only has sim users", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe"])
+	})
+	t.deepEqual(result, ["cadence.moe"])
+})
+
+test("getViaServers: also returns the most popular servers in order", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "thecollective.invalid", "selfhosted.invalid"])
+})
+
+test("getViaServers: does not return IP address servers", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:45.77.232.172:8443", "@cadence:[::1]:8443", "@cadence:123example.456example.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "123example.456example.invalid"])
+})
+
+test("getViaServers: also returns the highest power level user (100)", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@singleuser:selfhosted.invalid": 100,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@moderator:tractor.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "selfhosted.invalid", "thecollective.invalid", "tractor.invalid"])
+})
+
+test("getViaServers: also returns the highest power level user (50)", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@singleuser:selfhosted.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "tractor.invalid", "thecollective.invalid", "selfhosted.invalid"])
+})
+
+test("getViaServers: returns at most 4 results", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@singleuser:selfhosted.invalid": 100,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@cadence:123example.456example.invalid"])
+	})
+	t.deepEqual(result.length, 4)
+})
+
+test("getViaServers: returns results even when power levels can't be fetched", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => {
+			throw new Error("event not found or something")
+		},
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@cadence:123example.456example.invalid"])
+	})
+	t.deepEqual(result.length, 4)
+})
+
+test("getViaServers: only considers power levels of currently joined members", async t => {
+	const result = await getViaServers("!baby", {
+		getStateEvent: async () => ({
+			users: {
+				"@moderator:tractor.invalid": 50,
+				"@former_moderator:missing.invalid": 100,
+				"@_ooye_bot:cadence.moe": 100
+			}
+		}),
+		getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@singleuser:selfhosted.invalid"])
+	})
+	t.deepEqual(result, ["cadence.moe", "tractor.invalid", "thecollective.invalid", "selfhosted.invalid"])
+})

src/m2d/event-dispatcher.js

@@ -0,0 +1,194 @@
+// @ts-check
+
+/*
+ * Grab Matrix events we care about, check them, and bridge them.
+ */
+
+const util = require("util")
+const Ty = require("../types")
+const {discord, db, sync, as, select} = require("../passthrough")
+
+/** @type {import("./actions/send-event")} */
+const sendEvent = sync.require("./actions/send-event")
+/** @type {import("./actions/add-reaction")} */
+const addReaction = sync.require("./actions/add-reaction")
+/** @type {import("./actions/redact")} */
+const redact = sync.require("./actions/redact")
+/** @type {import("../matrix/matrix-command-handler")} */
+const matrixCommandHandler = sync.require("../matrix/matrix-command-handler")
+/** @type {import("./converters/utils")} */
+const utils = sync.require("./converters/utils")
+/** @type {import("../matrix/api")}) */
+const api = sync.require("../matrix/api")
+const {reg} = require("../matrix/read-registration")
+
+let lastReportedEvent = 0
+
+function guard(type, fn) {
+	return async function(event, ...args) {
+		try {
+			return await fn(event, ...args)
+		} catch (e) {
+			console.error("hit event-dispatcher's error handler with this exception:")
+			console.error(e) // TODO: also log errors into a file or into the database, maybe use a library for this? or just wing it?
+			console.error(`while handling this ${type} gateway event:`)
+			console.dir(event, {depth: null})
+
+			if (Date.now() - lastReportedEvent < 5000) return
+			lastReportedEvent = Date.now()
+
+			let stackLines = e.stack.split("\n")
+			api.sendEvent(event.room_id, "m.room.message", {
+				msgtype: "m.text",
+				body: "\u26a0 Matrix event not delivered to Discord. See formatted content for full details.",
+				format: "org.matrix.custom.html",
+				formatted_body: "\u26a0 <strong>Matrix event not delivered to Discord</strong>"
+					+ `<br>Event type: ${type}`
+					+ `<br>${e.toString()}`
+					+ `<br><details><summary>Error trace</summary>`
+					+ `<pre>${stackLines.join("\n")}</pre></details>`
+					+ `<details><summary>Original payload</summary>`
+					+ `<pre>${util.inspect(event, false, 4, false)}</pre></details>`,
+				"moe.cadence.ooye.error": {
+					source: "matrix",
+					payload: event
+				},
+				"m.mentions": {
+					user_ids: ["@cadence:cadence.moe"]
+				}
+			})
+		}
+	}
+}
+
+/**
+ * @param {Ty.Event.Outer<Ty.Event.M_Reaction>} reactionEvent
+ */
+async function onRetryReactionAdd(reactionEvent) {
+	const roomID = reactionEvent.room_id
+	const event = await api.getEvent(roomID, reactionEvent.content["m.relates_to"]?.event_id)
+
+	// Check that it's a real error from OOYE
+	const error = event.content["moe.cadence.ooye.error"]
+	if (event.sender !== `@${reg.sender_localpart}:${reg.ooye.server_name}` || !error) return
+
+	// To stop people injecting misleading messages, the reaction needs to come from either the original sender or a room moderator
+	if (reactionEvent.sender !== event.sender) {
+		// Check if it's a room moderator
+		const powerLevelsStateContent = await api.getStateEvent(roomID, "m.room.power_levels", "")
+		const powerLevel = powerLevelsStateContent.users?.[reactionEvent.sender] || 0
+		if (powerLevel < 50) return
+	}
+
+	// Retry
+	if (error.source === "matrix") {
+		as.emit(`type:${error.payload.type}`, error.payload)
+	} else if (error.source === "discord") {
+		discord.cloud.emit("event", error.payload)
+	}
+
+	// Redact the error to stop people from executing multiple retries
+	api.redactEvent(roomID, event.event_id)
+}
+
+sync.addTemporaryListener(as, "type:m.room.message", guard("m.room.message",
+/**
+ * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File} event it is a m.room.message because that's what this listener is filtering for
+ */
+async event => {
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	const messageResponses = await sendEvent.sendEvent(event)
+	if (event.type === "m.room.message" && event.content.msgtype === "m.text") {
+		// @ts-ignore
+		await matrixCommandHandler.execute(event)
+	}
+}))
+
+sync.addTemporaryListener(as, "type:m.sticker", guard("m.sticker",
+/**
+ * @param {Ty.Event.Outer_M_Sticker} event it is a m.sticker because that's what this listener is filtering for
+ */
+async event => {
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	const messageResponses = await sendEvent.sendEvent(event)
+}))
+
+sync.addTemporaryListener(as, "type:m.reaction", guard("m.reaction",
+/**
+ * @param {Ty.Event.Outer<Ty.Event.M_Reaction>} event it is a m.reaction because that's what this listener is filtering for
+ */
+async event => {
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	if (event.content["m.relates_to"].key === "🔁") {
+		// Try to bridge a failed event again?
+		await onRetryReactionAdd(event)
+	} else {
+		matrixCommandHandler.onReactionAdd(event)
+		await addReaction.addReaction(event)
+	}
+}))
+
+sync.addTemporaryListener(as, "type:m.room.redaction", guard("m.room.redaction",
+/**
+ * @param {Ty.Event.Outer_M_Room_Redaction} event it is a m.room.redaction because that's what this listener is filtering for
+ */
+async event => {
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	await redact.handle(event)
+}))
+
+sync.addTemporaryListener(as, "type:m.room.avatar", guard("m.room.avatar",
+/**
+ * @param {Ty.Event.StateOuter<Ty.Event.M_Room_Avatar>} event
+ */
+async event => {
+	if (event.state_key !== "") return
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	const url = event.content.url || null
+	db.prepare("UPDATE channel_room SET custom_avatar = ? WHERE room_id = ?").run(url, event.room_id)
+}))
+
+sync.addTemporaryListener(as, "type:m.room.name", guard("m.room.name",
+/**
+ * @param {Ty.Event.StateOuter<Ty.Event.M_Room_Name>} event
+ */
+async event => {
+	if (event.state_key !== "") return
+	if (utils.eventSenderIsFromDiscord(event.sender)) return
+	const name = event.content.name || null
+	db.prepare("UPDATE channel_room SET nick = ? WHERE room_id = ?").run(name, event.room_id)
+}))
+
+sync.addTemporaryListener(as, "type:m.room.member", guard("m.room.member",
+/**
+ * @param {Ty.Event.StateOuter<Ty.Event.M_Room_Member>} event
+ */
+async event => {
+	if (event.state_key[0] !== "@") return
+	if (utils.eventSenderIsFromDiscord(event.state_key)) return
+	if (event.content.membership === "leave" || event.content.membership === "ban") {
+		// Member is gone
+		db.prepare("DELETE FROM member_cache WHERE room_id = ? and mxid = ?").run(event.room_id, event.state_key)
+	} else {
+		// Member is here
+		db.prepare("INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES (?, ?, ?, ?) ON CONFLICT DO UPDATE SET displayname = ?, avatar_url = ?")
+			.run(
+				event.room_id, event.state_key,
+				event.content.displayname || null, event.content.avatar_url || null,
+				event.content.displayname || null, event.content.avatar_url || null
+			)
+	}
+}))
+
+sync.addTemporaryListener(as, "type:m.room.power_levels", guard("m.room.power_levels",
+/**
+ * @param {Ty.Event.StateOuter<Ty.Event.M_Power_Levels>} event
+ */
+async event => {
+	if (event.state_key !== "") return
+	const existingPower = select("member_cache", "mxid", {room_id: event.room_id}).pluck().all()
+	const newPower = event.content.users || {}
+	for (const mxid of existingPower) {
+		db.prepare("UPDATE member_cache SET power_level = ? WHERE room_id = ? AND mxid = ?").run(newPower[mxid] || 0, event.room_id, mxid)
+	}
+}))