From bf0691f9bb4a43b39db7c96d451e7d9218a5dc43 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Tue, 10 Oct 2023 14:03:53 +1300
Subject: [PATCH 001/480] Refactor reaction removals and add tests
---
d2m/actions/remove-reaction.js | 95 +++++---------
d2m/converters/remove-reaction.js | 88 +++++++++++++
d2m/converters/remove-reaction.test.js | 172 +++++++++++++++++++++++++
d2m/discord-packets.js | 10 +-
d2m/event-dispatcher.js | 49 +++----
test/test.js | 6 +-
6 files changed, 318 insertions(+), 102 deletions(-)
create mode 100644 d2m/converters/remove-reaction.js
create mode 100644 d2m/converters/remove-reaction.test.js
diff --git a/d2m/actions/remove-reaction.js b/d2m/actions/remove-reaction.js
index eec93a4..3047c32 100644
--- a/d2m/actions/remove-reaction.js
+++ b/d2m/actions/remove-reaction.js
@@ -1,7 +1,7 @@
// @ts-check
const Ty = require("../../types")
-const assert = require("assert").strict
+const DiscordTypes = require("discord-api-types/v10")
const passthrough = require("../../passthrough")
const {discord, sync, db, select} = passthrough
@@ -9,15 +9,15 @@ const {discord, sync, db, select} = passthrough
const api = sync.require("../../matrix/api")
/** @type {import("../converters/emoji-to-key")} */
const emojiToKey = sync.require("../converters/emoji-to-key")
-/** @type {import("../../m2d/converters/utils")} */
-const utils = sync.require("../../m2d/converters/utils")
/** @type {import("../../m2d/converters/emoji")} */
const emoji = sync.require("../../m2d/converters/emoji")
+/** @type {import("../converters/remove-reaction")} */
+const converter = sync.require("../converters/remove-reaction")
/**
- * @param {import("discord-api-types/v10").GatewayMessageReactionRemoveDispatchData} data
+ * @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData | DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData | DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data
*/
-async function removeReaction(data) {
+async function removeSomeReactions(data) {
const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
if (!roomID) return
const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, part: 0}).pluck().get()
@@ -25,76 +25,49 @@ async function removeReaction(data) {
/** @type {Ty.Pagination>} */
const relations = await api.getRelations(roomID, eventIDForMessage, "m.annotation")
- const key = await emojiToKey.emojiToKey(data.emoji)
- const wantToRemoveMatrixReaction = data.user_id === discord.application.id
- for (const event of relations.chunk) {
- if (event.content["m.relates_to"].key === key) {
- const lookingAtMatrixReaction = !utils.eventSenderIsFromDiscord(event.sender)
- if (lookingAtMatrixReaction && wantToRemoveMatrixReaction) {
- // We are removing a Matrix user's reaction, so we need to redact from the correct user ID (not @_ooye_matrix_bridge).
- // Even though the bridge bot only reacted once on Discord-side, multiple Matrix users may have
- // reacted on Matrix-side. Semantically, we want to remove the reaction from EVERY Matrix user.
- await api.redactEvent(roomID, event.event_id)
- // Clean up the database
- const hash = utils.getEventIDHash(event.event_id)
- db.prepare("DELETE FROM reaction WHERE hashed_event_id = ?").run(hash)
- }
- if (!lookingAtMatrixReaction && !wantToRemoveMatrixReaction) {
- // We are removing a Discord user's reaction, so we just make the sim user remove it.
- const mxid = select("sim", "mxid", {user_id: data.user_id}).pluck().get()
- if (mxid === event.sender) {
- await api.redactEvent(roomID, event.event_id, mxid)
- }
- }
- }
+ // Run the proper strategy and any strategy-specific database changes
+ const removals = await
+ ( "user_id" in data ? removeReaction(data, relations)
+ : "emoji" in data ? removeEmojiReaction(data, relations)
+ : removeAllReactions(data, relations))
+
+ // Redact the events and delete individual stored events in the database
+ for (const removal of removals) {
+ await api.redactEvent(roomID, removal.eventID, removal.mxid)
+ if (removal.hash) db.prepare("DELETE FROM reaction WHERE hashed_event_id = ?").run(removal.hash)
}
}
/**
- * @param {import("discord-api-types/v10").GatewayMessageReactionRemoveEmojiDispatchData} data
+ * @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData} data
+ * @param {Ty.Pagination>} relations
*/
-async function removeEmojiReaction(data) {
- const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
- if (!roomID) return
- const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, part: 0}).pluck().get()
- if (!eventIDForMessage) return
-
- /** @type {Ty.Pagination>} */
- const relations = await api.getRelations(roomID, eventIDForMessage, "m.annotation")
+async function removeReaction(data, relations) {
const key = await emojiToKey.emojiToKey(data.emoji)
+ return converter.removeReaction(data, relations, key)
+}
- for (const event of relations.chunk) {
- if (event.content["m.relates_to"].key === key) {
- const mxid = utils.eventSenderIsFromDiscord(event.sender) ? event.sender : undefined
- await api.redactEvent(roomID, event.event_id, mxid)
- }
- }
-
+/**
+ * @param {DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData} data
+ * @param {Ty.Pagination>} relations
+ */
+async function removeEmojiReaction(data, relations) {
+ const key = await emojiToKey.emojiToKey(data.emoji)
const discordPreferredEncoding = emoji.encodeEmoji(key, undefined)
db.prepare("DELETE FROM reaction WHERE message_id = ? AND encoded_emoji = ?").run(data.message_id, discordPreferredEncoding)
+
+ return converter.removeEmojiReaction(data, relations, key)
}
/**
- * @param {import("discord-api-types/v10").GatewayMessageReactionRemoveAllDispatchData} data
+ * @param {DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data
+ * @param {Ty.Pagination>} relations
*/
-async function removeAllReactions(data) {
- const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
- if (!roomID) return
- const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, part: 0}).pluck().get()
- if (!eventIDForMessage) return
-
- /** @type {Ty.Pagination>} */
- const relations = await api.getRelations(roomID, eventIDForMessage, "m.annotation")
-
- for (const event of relations.chunk) {
- const mxid = utils.eventSenderIsFromDiscord(event.sender) ? event.sender : undefined
- await api.redactEvent(roomID, event.event_id, mxid)
- }
-
+async function removeAllReactions(data, relations) {
db.prepare("DELETE FROM reaction WHERE message_id = ?").run(data.message_id)
+
+ return converter.removeAllReactions(data, relations)
}
-module.exports.removeReaction = removeReaction
-module.exports.removeEmojiReaction = removeEmojiReaction
-module.exports.removeAllReactions = removeAllReactions
+module.exports.removeSomeReactions = removeSomeReactions
diff --git a/d2m/converters/remove-reaction.js b/d2m/converters/remove-reaction.js
new file mode 100644
index 0000000..4fed269
--- /dev/null
+++ b/d2m/converters/remove-reaction.js
@@ -0,0 +1,88 @@
+// @ts-check
+
+const Ty = require("../../types")
+const DiscordTypes = require("discord-api-types/v10")
+
+const passthrough = require("../../passthrough")
+const {discord, sync, select} = passthrough
+/** @type {import("../../m2d/converters/utils")} */
+const utils = sync.require("../../m2d/converters/utils")
+
+/**
+ * @typedef ReactionRemoveRequest
+ * @prop {string} eventID
+ * @prop {string | null} mxid
+ * @prop {BigInt} [hash]
+ */
+
+/**
+ * @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData} data
+ * @param {Ty.Pagination>} relations
+ * @param {string} key
+ */
+function removeReaction(data, relations, key) {
+ /** @type {ReactionRemoveRequest[]} */
+ const removals = []
+
+ const wantToRemoveMatrixReaction = data.user_id === discord.application.id
+ for (const event of relations.chunk) {
+ const eventID = event.event_id
+ if (event.content["m.relates_to"].key === key) {
+ const lookingAtMatrixReaction = !utils.eventSenderIsFromDiscord(event.sender)
+ if (lookingAtMatrixReaction && wantToRemoveMatrixReaction) {
+ // We are removing a Matrix user's reaction, so we need to redact from the correct user ID (not @_ooye_matrix_bridge).
+ // Even though the bridge bot only reacted once on Discord-side, multiple Matrix users may have
+ // reacted on Matrix-side. Semantically, we want to remove the reaction from EVERY Matrix user.
+ // Also need to clean up the database.
+ const hash = utils.getEventIDHash(event.event_id)
+ removals.push({eventID, mxid: null, hash})
+ }
+ if (!lookingAtMatrixReaction && !wantToRemoveMatrixReaction) {
+ // We are removing a Discord user's reaction, so we just make the sim user remove it.
+ const mxid = select("sim", "mxid", {user_id: data.user_id}).pluck().get()
+ if (mxid === event.sender) {
+ removals.push({eventID, mxid})
+ }
+ }
+ }
+ }
+
+ return removals
+}
+
+/**
+ * @param {DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData} data
+ * @param {Ty.Pagination>} relations
+ * @param {string} key
+ */
+function removeEmojiReaction(data, relations, key) {
+ /** @type {ReactionRemoveRequest[]} */
+ const removals = []
+
+ for (const event of relations.chunk) {
+ const eventID = event.event_id
+ if (event.content["m.relates_to"].key === key) {
+ const mxid = utils.eventSenderIsFromDiscord(event.sender) ? event.sender : null
+ removals.push({eventID, mxid})
+ }
+ }
+
+ return removals
+}
+
+/**
+ * @param {DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data
+ * @param {Ty.Pagination>} relations
+ * @returns {ReactionRemoveRequest[]}
+ */
+function removeAllReactions(data, relations) {
+ return relations.chunk.map(event => {
+ const eventID = event.event_id
+ const mxid = utils.eventSenderIsFromDiscord(event.sender) ? event.sender : null
+ return {eventID, mxid}
+ })
+}
+
+module.exports.removeReaction = removeReaction
+module.exports.removeEmojiReaction = removeEmojiReaction
+module.exports.removeAllReactions = removeAllReactions
diff --git a/d2m/converters/remove-reaction.test.js b/d2m/converters/remove-reaction.test.js
new file mode 100644
index 0000000..a63721f
--- /dev/null
+++ b/d2m/converters/remove-reaction.test.js
@@ -0,0 +1,172 @@
+// @ts-check
+
+const {test} = require("supertape")
+const removeReaction = require("./remove-reaction")
+
+const BRIDGE_ID = "684280192553844747"
+
+function fakeSpecificReactionRemoval(userID, emoji, emojiID) {
+ return {
+ channel_id: "THE_CHANNEL",
+ message_id: "THE_MESSAGE",
+ user_id: userID,
+ emoji: {id: emojiID, name: emoji}
+ }
+}
+
+function fakeEmojiReactionRemoval(emoji, emojiID) {
+ return {
+ channel_id: "THE_CHANNEL",
+ message_id: "THE_MESSAGE",
+ emoji: {id: emojiID, name: emoji}
+ }
+}
+
+function fakeAllReactionRemoval() {
+ return {
+ channel_id: "THE_CHANNEL",
+ message_id: "THE_MESSAGE"
+ }
+}
+
+function fakeChunk(chunk) {
+ return {
+ chunk: chunk.map(({sender, key}, i) => ({
+ content: {
+ "m.relates_to": {
+ rel_type: "m.annotation",
+ event_id: "$message",
+ key
+ }
+ },
+ event_id: `$reaction_${i}`,
+ sender,
+ type: "m.reaction",
+ origin_server_ts: 0,
+ room_id: "!THE_ROOM",
+ unsigned: null
+ }))
+ }
+}
+
+test("remove reaction: a specific discord user's reaction is removed", t => {
+ const removals = removeReaction.removeReaction(
+ fakeSpecificReactionRemoval("820865262526005258", "🐈", null),
+ fakeChunk([{key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"}]),
+ "🐈"
+ )
+ t.deepEqual(removals, [{
+ eventID: "$reaction_0",
+ mxid: "@_ooye_crunch_god:cadence.moe"
+ }])
+})
+
+test("remove reaction: a specific matrix user's reaction is removed", t => {
+ const removals = removeReaction.removeReaction(
+ fakeSpecificReactionRemoval(BRIDGE_ID, "🐈", null),
+ fakeChunk([{key: "🐈", sender: "@cadence:cadence.moe"}]),
+ "🐈"
+ )
+ t.deepEqual(removals, [{
+ eventID: "$reaction_0",
+ mxid: null,
+ hash: 2842343637291700751n
+ }])
+})
+
+test("remove reaction: a specific discord user's reaction is removed when there are multiple reactions", t => {
+ const removals = removeReaction.removeReaction(
+ fakeSpecificReactionRemoval("820865262526005258", "🐈", null),
+ fakeChunk([
+ {key: "🐈⬛", sender: "@_ooye_crunch_god:cadence.moe"},
+ {key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"},
+ {key: "🐈", sender: "@_ooye_extremity:cadence.moe"},
+ {key: "🐈", sender: "@cadence:cadence.moe"},
+ {key: "🐈", sender: "@zoe:cadence.moe"}
+ ]),
+ "🐈"
+ )
+ t.deepEqual(removals, [{
+ eventID: "$reaction_1",
+ mxid: "@_ooye_crunch_god:cadence.moe"
+ }])
+})
+
+test("remove reaction: a specific reaction leads to all matrix users' reaction of the emoji being removed", t => {
+ const removals = removeReaction.removeReaction(
+ fakeSpecificReactionRemoval(BRIDGE_ID, "🐈", null),
+ fakeChunk([
+ {key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"},
+ {key: "🐈", sender: "@cadence:cadence.moe"},
+ {key: "🐈⬛", sender: "@zoe:cadence.moe"},
+ {key: "🐈", sender: "@zoe:cadence.moe"},
+ {key: "🐈", sender: "@_ooye_extremity:cadence.moe"}
+ ]),
+ "🐈"
+ )
+ t.deepEqual(removals, [{
+ eventID: "$reaction_1",
+ mxid: null,
+ hash: -8635141960139030904n
+ }, {
+ eventID: "$reaction_3",
+ mxid: null,
+ hash: 326222869084879263n
+ }])
+})
+
+test("remove reaction: an emoji removes all instances of the emoij from both sides", t => {
+ const removals = removeReaction.removeEmojiReaction(
+ fakeEmojiReactionRemoval("🐈", null),
+ fakeChunk([
+ {key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"},
+ {key: "🐈", sender: "@cadence:cadence.moe"},
+ {key: "🐈⬛", sender: "@zoe:cadence.moe"},
+ {key: "🐈", sender: "@zoe:cadence.moe"},
+ {key: "🐈", sender: "@_ooye_extremity:cadence.moe"}
+ ]),
+ "🐈"
+ )
+ t.deepEqual(removals, [{
+ eventID: "$reaction_0",
+ mxid: "@_ooye_crunch_god:cadence.moe"
+ }, {
+ eventID: "$reaction_1",
+ mxid: null
+ }, {
+ eventID: "$reaction_3",
+ mxid: null
+ }, {
+ eventID: "$reaction_4",
+ mxid: "@_ooye_extremity:cadence.moe"
+ }])
+})
+
+test("remove reaction: remove all removes all from both sides", t => {
+ const removals = removeReaction.removeAllReactions(
+ fakeAllReactionRemoval(),
+ fakeChunk([
+ {key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"},
+ {key: "🐈", sender: "@cadence:cadence.moe"},
+ {key: "🐈⬛", sender: "@zoe:cadence.moe"},
+ {key: "🐈", sender: "@zoe:cadence.moe"},
+ {key: "🐈", sender: "@_ooye_extremity:cadence.moe"}
+ ])
+ )
+ t.deepEqual(removals, [{
+ eventID: "$reaction_0",
+ mxid: "@_ooye_crunch_god:cadence.moe"
+ }, {
+ eventID: "$reaction_1",
+ mxid: null
+ }, {
+ eventID: "$reaction_2",
+ mxid: null
+ }, {
+ eventID: "$reaction_3",
+ mxid: null
+ }, {
+ eventID: "$reaction_4",
+ mxid: "@_ooye_extremity:cadence.moe"
+ }])
+})
diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js
index dfba8b0..83c31cd 100644
--- a/d2m/discord-packets.js
+++ b/d2m/discord-packets.js
@@ -155,14 +155,8 @@ const utils = {
} else if (message.t === "MESSAGE_REACTION_ADD") {
await eventDispatcher.onReactionAdd(client, message.d)
- } else if (message.t === "MESSAGE_REACTION_REMOVE") {
- await eventDispatcher.onReactionRemove(client, message.d)
-
- } else if (message.t === "MESSAGE_REACTION_REMOVE_EMOJI") {
- await eventDispatcher.onReactionEmojiRemove(client, message.d)
-
- } else if (message.t === "MESSAGE_REACTION_REMOVE_ALL") {
- await eventDispatcher.onRemoveAllReactions(client, message.d)
+ } else if (message.t === "MESSAGE_REACTION_REMOVE" || message.t === "MESSAGE_REACTION_REMOVE_EMOJI" || message.t === "MESSAGE_REACTION_REMOVE_ALL") {
+ await eventDispatcher.onSomeReactionsRemoved(client, message.d)
}
} catch (e) {
// Let OOYE try to handle errors too
diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js
index 82e3b64..0f9f1e6 100644
--- a/d2m/event-dispatcher.js
+++ b/d2m/event-dispatcher.js
@@ -1,4 +1,5 @@
const assert = require("assert").strict
+const DiscordTypes = require("discord-api-types/v10")
const util = require("util")
const {sync, db, select, from} = require("../passthrough")
@@ -80,7 +81,7 @@ module.exports = {
* If more messages were missed, only the latest missed message will be posted. TODO: Consider bridging more, or post a warning when skipping history?
* This can ONLY detect new messages, not any other kind of event. Any missed edits, deletes, reactions, etc will not be bridged.
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayGuildCreateDispatchData} guild
+ * @param {DiscordTypes.GatewayGuildCreateDispatchData} guild
*/
async checkMissedMessages(client, guild) {
if (guild.unavailable) return
@@ -126,7 +127,7 @@ module.exports = {
* Announces to the parent room that the thread room has been created.
* See notes.md, "Ignore MESSAGE_UPDATE and bridge THREAD_CREATE as the announcement"
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").APIThreadChannel} thread
+ * @param {DiscordTypes.APIThreadChannel} thread
*/
async onThreadCreate(client, thread) {
const parentRoomID = select("channel_room", "room_id", {channel_id: thread.parent_id}).pluck().get()
@@ -137,7 +138,7 @@ module.exports = {
/**
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayGuildUpdateDispatchData} guild
+ * @param {DiscordTypes.GatewayGuildUpdateDispatchData} guild
*/
async onGuildUpdate(client, guild) {
const spaceID = select("guild_space", "space_id", {guild_id: guild.id}).pluck().get()
@@ -147,7 +148,7 @@ module.exports = {
/**
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayChannelUpdateDispatchData} channelOrThread
+ * @param {DiscordTypes.GatewayChannelUpdateDispatchData} channelOrThread
* @param {boolean} isThread
*/
async onChannelOrThreadUpdate(client, channelOrThread, isThread) {
@@ -158,7 +159,7 @@ module.exports = {
/**
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message
+ * @param {DiscordTypes.GatewayMessageCreateDispatchData} message
*/
async onMessageCreate(client, message) {
if (message.author.username === "Deleted User") return // Nothing we can do for deleted users.
@@ -169,7 +170,7 @@ module.exports = {
return
}
}
- /** @type {import("discord-api-types/v10").APIGuildChannel} */
+ /** @type {DiscordTypes.APIGuildChannel} */
const channel = client.channels.get(message.channel_id)
if (!channel.guild_id) return // Nothing we can do in direct messages.
const guild = client.guilds.get(channel.guild_id)
@@ -180,7 +181,7 @@ module.exports = {
/**
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayMessageUpdateDispatchData} data
+ * @param {DiscordTypes.GatewayMessageUpdateDispatchData} data
*/
async onMessageUpdate(client, data) {
if (data.webhook_id) {
@@ -193,9 +194,9 @@ module.exports = {
// Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes.
// If the message content is a string then it includes all interesting fields and is meaningful.
if (typeof data.content === "string") {
- /** @type {import("discord-api-types/v10").GatewayMessageCreateDispatchData} */
+ /** @type {DiscordTypes.GatewayMessageCreateDispatchData} */
const message = data
- /** @type {import("discord-api-types/v10").APIGuildChannel} */
+ /** @type {DiscordTypes.APIGuildChannel} */
const channel = client.channels.get(message.channel_id)
if (!channel.guild_id) return // Nothing we can do in direct messages.
const guild = client.guilds.get(channel.guild_id)
@@ -205,7 +206,7 @@ module.exports = {
/**
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayMessageReactionAddDispatchData} data
+ * @param {DiscordTypes.GatewayMessageReactionAddDispatchData} data
*/
async onReactionAdd(client, data) {
if (data.user_id === client.user.id) return // m2d reactions are added by the discord bot user - do not reflect them back to matrix.
@@ -215,31 +216,15 @@ module.exports = {
/**
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayMessageReactionRemoveDispatchData} data
+ * @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData | DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData | DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data
*/
- async onReactionRemove(client, data) {
- await removeReaction.removeReaction(data)
+ async onSomeReactionsRemoved(client, data) {
+ await removeReaction.removeSomeReactions(data)
},
/**
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayMessageReactionRemoveEmojiDispatchData} data
- */
- async onReactionEmojiRemove(client, data) {
- await removeReaction.removeEmojiReaction(data)
- },
-
- /**
- * @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayMessageReactionRemoveAllDispatchData} data
- */
- async onRemoveAllReactions(client, data) {
- await removeReaction.removeAllReactions(data)
- },
-
- /**
- * @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayMessageDeleteDispatchData} data
+ * @param {DiscordTypes.GatewayMessageDeleteDispatchData} data
*/
async onMessageDelete(client, data) {
await deleteMessage.deleteMessage(data)
@@ -247,7 +232,7 @@ module.exports = {
/**
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayTypingStartDispatchData} data
+ * @param {DiscordTypes.GatewayTypingStartDispatchData} data
*/
async onTypingStart(client, data) {
const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
@@ -262,7 +247,7 @@ module.exports = {
/**
* @param {import("./discord-client")} client
- * @param {import("discord-api-types/v10").GatewayGuildEmojisUpdateDispatchData | import("discord-api-types/v10").GatewayGuildStickersUpdateDispatchData} data
+ * @param {DiscordTypes.GatewayGuildEmojisUpdateDispatchData | DiscordTypes.GatewayGuildStickersUpdateDispatchData} data
*/
async onExpressionsUpdate(client, data) {
await createSpace.syncSpaceExpressions(data)
diff --git a/test/test.js b/test/test.js
index 98ecf1c..5cc851e 100644
--- a/test/test.js
+++ b/test/test.js
@@ -20,7 +20,10 @@ const sync = new HeatSync({watchFS: false})
const discord = {
guilds: new Map([
[data.guild.general.id, data.guild.general]
- ])
+ ]),
+ application: {
+ id: "684280192553844747"
+ }
}
Object.assign(passthrough, { discord, config, sync, db })
@@ -49,6 +52,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
require("../d2m/converters/message-to-event.test")
require("../d2m/converters/message-to-event.embeds.test")
require("../d2m/converters/edit-to-changes.test")
+ require("../d2m/converters/remove-reaction.test")
require("../d2m/converters/thread-to-announcement.test")
require("../d2m/converters/user-to-mxid.test")
require("../d2m/converters/emoji-to-key.test")
From 48d69c053949eed8526e58338ec7b30f444c71ae Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Tue, 10 Oct 2023 17:41:53 +1300
Subject: [PATCH 002/480] Start work on pinned events
---
d2m/actions/update-pins.js | 22 ++
d2m/converters/pins-to-list.js | 18 ++
d2m/converters/pins-to-list.test.js | 12 +
d2m/discord-packets.js | 3 +
d2m/event-dispatcher.js | 12 +
docs/how-to-add-a-new-event-type.md | 349 ++++++++++++++++++++++++++++
test/data.js | 8 +
test/ooye-test-data.sql | 4 +-
test/test.js | 1 +
9 files changed, 427 insertions(+), 2 deletions(-)
create mode 100644 d2m/actions/update-pins.js
create mode 100644 d2m/converters/pins-to-list.js
create mode 100644 d2m/converters/pins-to-list.test.js
create mode 100644 docs/how-to-add-a-new-event-type.md
diff --git a/d2m/actions/update-pins.js b/d2m/actions/update-pins.js
new file mode 100644
index 0000000..40cc358
--- /dev/null
+++ b/d2m/actions/update-pins.js
@@ -0,0 +1,22 @@
+// @ts-check
+
+const passthrough = require("../../passthrough")
+const {discord, sync} = passthrough
+/** @type {import("../converters/pins-to-list")} */
+const pinsToList = sync.require("../converters/pins-to-list")
+/** @type {import("../../matrix/api")} */
+const api = sync.require("../../matrix/api")
+
+/**
+ * @param {string} channelID
+ * @param {string} roomID
+ */
+async function updatePins(channelID, roomID) {
+ const pins = await discord.snow.channel.getChannelPinnedMessages(channelID)
+ const eventIDs = pinsToList.pinsToList(pins)
+ await api.sendState(roomID, "m.room.pinned_events", "", {
+ pinned: eventIDs
+ })
+}
+
+module.exports.updatePins = updatePins
diff --git a/d2m/converters/pins-to-list.js b/d2m/converters/pins-to-list.js
new file mode 100644
index 0000000..f401de2
--- /dev/null
+++ b/d2m/converters/pins-to-list.js
@@ -0,0 +1,18 @@
+// @ts-check
+
+const {select} = require("../../passthrough")
+
+/**
+ * @param {import("discord-api-types/v10").RESTGetAPIChannelPinsResult} pins
+ */
+function pinsToList(pins) {
+ /** @type {string[]} */
+ const result = []
+ for (const message of pins) {
+ const eventID = select("event_message", "event_id", {message_id: message.id, part: 0}).pluck().get()
+ if (eventID) result.push(eventID)
+ }
+ return result
+}
+
+module.exports.pinsToList = pinsToList
diff --git a/d2m/converters/pins-to-list.test.js b/d2m/converters/pins-to-list.test.js
new file mode 100644
index 0000000..c2e3774
--- /dev/null
+++ b/d2m/converters/pins-to-list.test.js
@@ -0,0 +1,12 @@
+const {test} = require("supertape")
+const data = require("../../test/data")
+const {pinsToList} = require("./pins-to-list")
+
+test("pins2list: converts known IDs, ignores unknown IDs", t => {
+ const result = pinsToList(data.pins.faked)
+ t.deepEqual(result, [
+ "$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg",
+ "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA",
+ "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4"
+ ])
+})
diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js
index 83c31cd..4de84d9 100644
--- a/d2m/discord-packets.js
+++ b/d2m/discord-packets.js
@@ -133,6 +133,9 @@ const utils = {
} else if (message.t === "CHANNEL_UPDATE") {
await eventDispatcher.onChannelOrThreadUpdate(client, message.d, false)
+ } else if (message.t === "CHANNEL_PINS_UPDATE") {
+ await eventDispatcher.onChannelPinsUpdate(client, message.d)
+
} else if (message.t === "THREAD_CREATE") {
// @ts-ignore
await eventDispatcher.onThreadCreate(client, message.d)
diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js
index 0f9f1e6..12a3507 100644
--- a/d2m/event-dispatcher.js
+++ b/d2m/event-dispatcher.js
@@ -19,6 +19,8 @@ const announceThread = sync.require("./actions/announce-thread")
const createRoom = sync.require("./actions/create-room")
/** @type {import("./actions/create-space")}) */
const createSpace = sync.require("./actions/create-space")
+/** @type {import("./actions/update-pins")}) */
+const updatePins = sync.require("./actions/update-pins")
/** @type {import("../matrix/api")}) */
const api = sync.require("../matrix/api")
/** @type {import("../discord/discord-command-handler")}) */
@@ -157,6 +159,16 @@ module.exports = {
await createRoom.syncRoom(channelOrThread.id)
},
+ /**
+ * @param {import("./discord-client")} client
+ * @param {DiscordTypes.GatewayChannelPinsUpdateDispatchData} data
+ */
+ async onChannelPinsUpdate(client, data) {
+ const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
+ if (!roomID) return // No target room to update pins in
+ await updatePins.updatePins(data.channel_id, roomID)
+ },
+
/**
* @param {import("./discord-client")} client
* @param {DiscordTypes.GatewayMessageCreateDispatchData} message
diff --git a/docs/how-to-add-a-new-event-type.md b/docs/how-to-add-a-new-event-type.md
new file mode 100644
index 0000000..6304865
--- /dev/null
+++ b/docs/how-to-add-a-new-event-type.md
@@ -0,0 +1,349 @@
+# How to add a new event type
+
+It's much easier to understand code with examples. So let's go through it together and add support for **pinned events** to Out Of Your Element.
+
+## Gathering intel
+
+First, we need to know what pinned events are supposed to look like. The Matrix C-S spec gives this example:
+
+> **pinned** ... [string] ... Required: An ordered list of event IDs to pin.
+> ```json
+> {
+> "content": {
+> "pinned": [
+> "$someevent:example.org"
+> ]
+> },
+> "event_id": "$143273582443PhrSn:example.org",
+> "room_id": "!jEsUZKDJdhlrceRyVU:example.org",
+> "sender": "@example:example.org",
+> "state_key": "",
+> "type": "m.room.pinned_events",
+> }
+> ```
+
+This is part of the persistent room state. Simple enough. To update this, the state event and its list of pinned events is updated as a whole.
+
+What does it look like on Discord-side?
+
+> **Get Pinned Messages** \
+> `GET` `/channels/{channel.id}/pins` \
+> Returns all pinned messages in the channel as an array of message objects.
+
+This is an API request to get the pinned messages. To update this, an API request will pin or unpin any specific message, adding or removing it from the list.
+
+## What will the converter look like?
+
+The converter will be very different in both directions.
+
+For d2m, we will get the list of pinned messages, we will convert each message ID into the ID of an event we already have, and then we will set the entire `m.room.pinned_events` state to that list.
+
+For m2d, we will have to diff the list of pinned messages against the previous version of the list, and for each event that was pinned or unpinned, we will send an API request to Discord to change its state.
+
+## Missing messages
+
+> ...we will convert each message ID into the ID of an event ***we already have***
+
+As a couple of examples, the message might not be bridged if it was sent before OOYE was set up, or if OOYE just had a bug trying to handle that message. If a particular Discord message wasn't bridged, and then it gets pinned, we're in a bit of a pickle. We can't pin the Matrix equivalent because it doesn't exist.
+
+In this situation we need to stop and think about the possible paths forward we could take.
+
+* We could ignore this message and just not pin it.
+* We could convert and send this message now with its original timestamp from the past, then pin this representation.
+
+The latter method would still make the message appear at the bottom of the timeline for most Matrix clients, since for most the timestamp doesn't determine the actual _order._ It would then be confusing why an odd message suddenly appeared, because a pins change isn't that noticable in the room.
+
+To avoid this problem, I'll just go with the former method and ignore the message, so Matrix will only have some of the pins that Discord has. We will need to watch out if a Matrix user edits this list of partial pins, because if we _only_ pinned things on Discord that were pinned on Matrix, those partial pins Discord would be lost from Discord side.
+
+In this situation I will prefer to keep the pins list inconsistent between both sides and only bridge _changes_ to the list.
+
+If you were implementing this for real, you might have made different decisions than me, and that's okay. It's a matter of taste. You just need to be aware of the consequences of what you choose.
+
+## Test data for the d2m converter
+
+Let's start writing the d2m converter. It's helpful to write unit tests for Out Of Your Element, since this lets you check if it worked without having to start up a local copy of the bridge or play around with the interface.
+
+Normally for getting test data, I would `curl` the Discord API to grab some real data and put it into `data.js` (and possibly also `ooye-test-data.sql`. But this time, I'll fabricate some test data. Here it is:
+
+```js
+[
+ {id: "1126786462646550579"},
+ {id: "1141501302736695316"},
+ {id: "1106366167788044450"},
+ {id: "1115688611186193400"}
+]
+```
+
+"These aren't message objects!" I hear you cry. Correct. I already know that my implementation is not going to care about any properties on these message object other than the IDs, so I'm just making a list of IDs to save time.
+
+These IDs were carefully chosen. The first three are already in `ooye-test-data.sql` and are associated with event IDs. This is great, because in our test case, the Discord IDs will be converted to those event IDs. The fourth ID doesn't exist on Matrix-side. This is to test that partial pins are handled as expected, like I wrote in the previous section.
+
+Now that I've got my list, I will make my first change to the code. I will add these IDs to `test/data.js`:
+
+```diff
+diff --git a/test/data.js b/test/data.js
+index c36f252..4919beb 100644
+--- a/test/data.js
++++ b/test/data.js
+@@ -221,6 +221,14 @@ module.exports = {
+ deaf: false
+ }
+ },
++ pins: {
++ faked: [
++ {id: "1126786462646550579"},
++ {id: "1141501302736695316"},
++ {id: "1106366167788044450"},
++ {id: "1115688611186193400"}
++ ]
++ },
+ message: {
+ // Display order is text content, attachments, then stickers
+ simple_plaintext: {
+```
+
+## Writing the d2m converter
+
+We can write a function that operates on this data to convert it to events. This is a _converter,_ not an _action._ it won't _do_ anything by itself. So it goes in the converters folder. The actual function is pretty simple since I've already planned what to do:
+
+```diff
+diff --git a/d2m/converters/pins-to-list.js b/d2m/converters/pins-to-list.js
+new file mode 100644
+index 0000000..e4107be
+--- /dev/null
++++ b/d2m/converters/pins-to-list.js
+@@ -0,0 +1,18 @@
++// @ts-check
++
++const {select} = require("../../passthrough")
++
++/**
++ * @param {import("discord-api-types/v10").RESTGetAPIChannelPinsResult} pins
++ */
++function pinsToList(pins) {
++ /** @type {string[]} */
++ const result = []
++ for (const message of pins) {
++ const eventID = select("event_message", "event_id", {message_id: message.id}).pluck().get()
++ if (eventID) result.push(eventID)
++ }
++ return result
++}
++
++module.exports.pinsToList = pinsToList
+```
+
+## Test case for the d2m converter
+
+There's not much room for bugs in this function. A single manual test that it works would be good enough for me. But since this is an example of how you can add your own, let's add a test case for this. We'll take the data we just prepared and process it through the function we just wrote:
+
+```diff
+diff --git a/d2m/converters/pins-to-list.test.js b/d2m/converters/pins-to-list.test.js
+new file mode 100644
+index 0000000..c2e3774
+--- /dev/null
++++ b/d2m/converters/pins-to-list.test.js
+@@ -0,0 +1,12 @@
++const {test} = require("supertape")
++const data = require("../../test/data")
++const {pinsToList} = require("./pins-to-list")
++
++test("pins2list: converts known IDs, ignores unknown IDs", t => {
++ const result = pinsToList(data.pins.faked)
++ t.deepEqual(result, [
++ "$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg",
++ "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA",
++ "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4"
++ ])
++})
+```
+
+Don't forget to list your test in `test.js` so that it gets picked up:
+
+```diff
+diff --git a/test/test.js b/test/test.js
+index 5cc851e..280503d 100644
+--- a/test/test.js
++++ b/test/test.js
+@@ -52,6 +52,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
+ require("../d2m/converters/message-to-event.test")
+ require("../d2m/converters/message-to-event.embeds.test")
+ require("../d2m/converters/edit-to-changes.test")
++ require("../d2m/converters/pins-to-list.test")
+ require("../d2m/converters/remove-reaction.test")
+ require("../d2m/converters/thread-to-announcement.test")
+ require("../d2m/converters/user-to-mxid.test")
+```
+
+Good to go.
+
+```
+><> $ npm t
+
+> out-of-your-element@1.1.0 test
+> cross-env FORCE_COLOR=true supertape --no-check-assertions-count --format tap test/test.js | tap-dot
+
+ pins2list: converts known IDs, ignores unknown IDs - should deep equal
+ operator: deepEqual
+ diff: |-
+ Array [
+ "$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg",
+ - "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA",
+ - "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4",
+ + "$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI",
+ + "$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU",
+ ]
+ at out-of-your-element/d2m/converters/pins-to-list.test.js:7:4
+ Error: should deep equal
+ at run (file:///out-of-your-element/node_modules/supertape/lib/operators.mjs:272:33)
+ at Object.deepEqual (file:///out-of-your-element/node_modules/supertape/lib/operators.mjs:198:9)
+ at out-of-your-element/d2m/converters/pins-to-list.test.js:7:4
+ at module.exports (out-of-your-element/node_modules/try-to-catch/lib/try-to-catch.js:7:29)
+```
+
+Oh, this was actually an accident, I didn't make it fail for demonstration purposes! Let's see what this bug is. It's returning the right number of IDs, but 2 out of the 3 are incorrect. The green `-` lines are "expected" and the red `+` lines are "actual". I should check where that wrong ID `$51f...` got taken from.
+
+```
+ - snip - ooye-test-data.sql
+('$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA', 'm.room.message', 'm.text', '1141501302736695316', 0, 1),
+('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', 1, 1),
+```
+
+Context: This Discord message `1141501302736695316` is actually part of 2 different Matrix events, `$mtR...` and `$51f...`. This often happens when a Discord user uploads an image with a caption. Matrix doesn't support combined image+text events, so the image and the text have to be bridged to separate events. We should consider the text to be the primary part, and pin that, and consider the image to be the secondary part, and not pin that.
+
+We already have a column `part` in the `event_message` table for this reason! When `part = 0`, that's the primary part. I'll edit the converter to actually use that column:
+
+```diff
+diff --git a/d2m/converters/pins-to-list.js b/d2m/converters/pins-to-list.js
+index e4107be..f401de2 100644
+--- a/d2m/converters/pins-to-list.js
++++ b/d2m/converters/pins-to-list.js
+@@ -9,7 +9,7 @@ function pinsToList(pins) {
+ /** @type {string[]} */
+ const result = []
+ for (const message of pins) {
+- const eventID = select("event_message", "event_id", {message_id: message.id}).pluck().get()
++ const eventID = select("event_message", "event_id", {message_id: message.id, part: 0}).pluck().get()
+ if (eventID) result.push(eventID)
+ }
+ return result
+```
+
+```
+><> $ npm t
+
+ 144 tests
+ 232 passed
+
+ Pass!
+```
+
+## Wiring it up to an action
+
+Actions call converters to do the thing, but actions have to receive their input event from somewhere. Let's wire it up so we receive a "pins changed" event from Discord and do the whole flow from there. Checking the documentation again, Discord will trigger this gateway event when the pins change:
+
+> **Channel Pins Update** \
+> Sent when a message is pinned or unpinned in a text channel. This is not sent when a pinned message is deleted. \
+> **guild_id?** ... snowflake ... ID of the guild \
+> **channel_id** ... snowflake ... ID of the channel \
+> **last_pin_timestamp?** ... ?ISO8601 timestamp ... Time at which the most recent pinned message was pinned
+
+Notably, the event doesn't deliver the actual list of pinned messages to us. We'll have to listen for this event, then trigger an API request to `GET` the pins list. Alright, enough preparation, time to code.
+
+All packets are delivered to `discord-packets.js` which manages the internal state of the Discord object and then passes it on to a function in `event-dispatcher.js`:
+
+```diff
+diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js
+index 83c31cd..4de84d9 100644
+--- a/d2m/discord-packets.js
++++ b/d2m/discord-packets.js
+@@ -133,6 +133,9 @@ const utils = {
+ } else if (message.t === "CHANNEL_UPDATE") {
+ await eventDispatcher.onChannelOrThreadUpdate(client, message.d, false)
+
++ } else if (message.t === "CHANNEL_PINS_UPDATE") {
++ await eventDispatcher.onChannelPinsUpdate(client, message.d)
++
+ } else if (message.t === "THREAD_CREATE") {
+ // @ts-ignore
+ await eventDispatcher.onThreadCreate(client, message.d)
+```
+
+`event-dispatcher.js` will now check if the event seems reasonable and is allowed in this context. For example, we can only update pins if the channel is actually bridged somewhere. This should be another quick check which passes to an action to do the API calls:
+
+```diff
+diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js
+index 0f9f1e6..6e91e9e 100644
+--- a/d2m/event-dispatcher.js
++++ b/d2m/event-dispatcher.js
+@@ -19,6 +19,8 @@ const announceThread = sync.require("./actions/announce-thread")
+ const createRoom = sync.require("./actions/create-room")
+ /** @type {import("./actions/create-space")}) */
+ const createSpace = sync.require("./actions/create-space")
++/** @type {import("./actions/update-pins")}) */
++const updatePins = sync.require("./actions/update-pins")
+ /** @type {import("../matrix/api")}) */
+ const api = sync.require("../matrix/api")
+ /** @type {import("../discord/discord-command-handler")}) */
+@@ -157,6 +159,16 @@ module.exports = {
+ await createRoom.syncRoom(channelOrThread.id)
+ },
+
++ /**
++ * @param {import("./discord-client")} client
++ * @param {DiscordTypes.GatewayChannelPinsUpdateDispatchData} data
++ */
++ async onChannelPinsUpdate(client, data) {
++ const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
++ if (!roomID) return // No target room to update pins in
++ await updatePins.updatePins(data.channel_id, roomID)
++ },
++
+ /**
+ * @param {import("./discord-client")} client
+ * @param {DiscordTypes.GatewayMessageCreateDispatchData} message
+```
+
+And now I can create the `update-pins.js` action:
+
+```diff
+diff --git a/d2m/actions/update-pins.js b/d2m/actions/update-pins.js
+new file mode 100644
+index 0000000..40cc358
+--- /dev/null
++++ b/d2m/actions/update-pins.js
+@@ -0,0 +1,22 @@
++// @ts-check
++
++const passthrough = require("../../passthrough")
++const {discord, sync} = passthrough
++/** @type {import("../converters/pins-to-list")} */
++const pinsToList = sync.require("../converters/pins-to-list")
++/** @type {import("../../matrix/api")} */
++const api = sync.require("../../matrix/api")
++
++/**
++ * @param {string} channelID
++ * @param {string} roomID
++ */
++async function updatePins(channelID, roomID) {
++ const pins = await discord.snow.channel.getChannelPinnedMessages(channelID)
++ const eventIDs = pinsToList.pinsToList(pins)
++ await api.sendState(roomID, "m.room.pinned_events", "", {
++ pinned: eventIDs
++ })
++}
++
++module.exports.updatePins = updatePins
+```
+
+I try to keep as much logic as possible out of the actions, meaning I never have to unit test the actions themselves, and I can skip right ahead to trying this with the real bot.
+
+## Notes on missed events
+
+Note that this will only sync pins _when the pins change._ Existing pins from Discord will not be backfilled to Matrix rooms. If I wanted, there's a couple of ways I could address this:
+
+* I could create a one-shot script in `scripts/update-pins.js` which will sync pins for _all_ Discord channels right away. I can run this after finishing the feature, or if the bot has been offline for some time.
+* I could create a database table that holds the timestamp of the most recently detected pin for each channel - the `last_pin_timestamp` field from the gateway. Every time the bot starts, it would automatically compare the database table against every channel, and if the pins have changed since it last looked, it could automatically update them.
+
+I already have a mechanism for backfilling missed messages when the bridge starts up. Option 2 there would add a similar feature for backfilling missed pins. That could be worth considering, but it's less important and more complex. Perhaps we'll come back to it.
diff --git a/test/data.js b/test/data.js
index c36f252..4919beb 100644
--- a/test/data.js
+++ b/test/data.js
@@ -221,6 +221,14 @@ module.exports = {
deaf: false
}
},
+ pins: {
+ faked: [
+ {id: "1126786462646550579"},
+ {id: "1141501302736695316"},
+ {id: "1106366167788044450"},
+ {id: "1115688611186193400"}
+ ]
+ },
message: {
// Display order is text content, attachments, then stickers
simple_plaintext: {
diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql
index 68932dd..1ce0467 100644
--- a/test/ooye-test-data.sql
+++ b/test/ooye-test-data.sql
@@ -51,8 +51,8 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part
('$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q', 'm.room.message', 'm.text', '1128084748338741392', 0, 1),
('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', 0, 1),
('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', 0, 1),
-('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 0, 0),
-('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 0, 0),
+('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 1, 0),
+('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 1, 0),
('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', 0, 1),
('$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8', 'm.room.message', 'm.text', '1144874214311067708', 0, 0),
('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs', 'm.room.message', 'm.text', '1145688633186193479', 0, 0),
diff --git a/test/test.js b/test/test.js
index 5cc851e..280503d 100644
--- a/test/test.js
+++ b/test/test.js
@@ -52,6 +52,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
require("../d2m/converters/message-to-event.test")
require("../d2m/converters/message-to-event.embeds.test")
require("../d2m/converters/edit-to-changes.test")
+ require("../d2m/converters/pins-to-list.test")
require("../d2m/converters/remove-reaction.test")
require("../d2m/converters/thread-to-announcement.test")
require("../d2m/converters/user-to-mxid.test")
From 6bbfbd072193b9709d44bb759d2ad63218bc9228 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Wed, 11 Oct 2023 09:46:11 +1300
Subject: [PATCH 003/480] Red question mark emoji encoding should be trimmed
---
m2d/converters/emoji.js | 1 +
1 file changed, 1 insertion(+)
diff --git a/m2d/converters/emoji.js b/m2d/converters/emoji.js
index e17367c..214022f 100644
--- a/m2d/converters/emoji.js
+++ b/m2d/converters/emoji.js
@@ -38,6 +38,7 @@ function encodeEmoji(input, shortcode) {
"%F0%9F%91%8D", // 👍
"%E2%AD%90", // ⭐
"%F0%9F%90%88", // 🐈
+ "%E2%9D%93", // ❓
]
discordPreferredEncoding =
From 5c41b95919582999e8c2aa4dedf3c94bd8e80497 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Wed, 11 Oct 2023 11:27:50 +1300
Subject: [PATCH 004/480] Clarify documentation note
---
docs/how-to-add-a-new-event-type.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/how-to-add-a-new-event-type.md b/docs/how-to-add-a-new-event-type.md
index 6304865..10beec1 100644
--- a/docs/how-to-add-a-new-event-type.md
+++ b/docs/how-to-add-a-new-event-type.md
@@ -337,7 +337,7 @@ index 0000000..40cc358
+module.exports.updatePins = updatePins
```
-I try to keep as much logic as possible out of the actions, meaning I never have to unit test the actions themselves, and I can skip right ahead to trying this with the real bot.
+I try to keep as much logic as possible out of the actions and in the converters. This should mean I *never have to unit test the actions themselves.* The actions will be tested manually with the real bot.
## Notes on missed events
From 613a1dc0866067f597b691b71d07bb72e70dfda5 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Thu, 12 Oct 2023 20:30:41 +1300
Subject: [PATCH 005/480] Add private/linkable/public privacy rules
---
d2m/actions/create-room.js | 61 +++++++++++++++------
d2m/actions/create-space.js | 37 +++++++------
d2m/actions/send-message.js | 4 +-
db/migrations/0006-add-privacy-to-space.sql | 5 ++
db/orm-defs.d.ts | 1 +
test/ooye-test-data.sql | 4 +-
6 files changed, 76 insertions(+), 36 deletions(-)
create mode 100644 db/migrations/0006-add-privacy-to-space.sql
diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js
index 51926ee..bee45e2 100644
--- a/d2m/actions/create-room.js
+++ b/d2m/actions/create-room.js
@@ -15,6 +15,24 @@ const ks = sync.require("../../matrix/kstate")
/** @type {import("./create-space")}) */
const createSpace = sync.require("./create-space") // watch out for the require loop
+/**
+ * There are 3 levels of room privacy:
+ * 0: Room is invite-only.
+ * 1: Anybody can use a link to join.
+ * 2: Room is published in room directory.
+ */
+const PRIVACY_ENUMS = {
+ PRESET: ["private_chat", "public_chat", "public_chat"],
+ VISIBILITY: ["private", "private", "public"],
+ SPACE_HISTORY_VISIBILITY: ["invited", "world_readable", "world_readable"], // copying from element client
+ ROOM_HISTORY_VISIBILITY: ["shared", "shared", "world_readable"], // any events sent after are visible, but for world_readable anybody can read without even joining
+ GUEST_ACCESS: ["can_join", "forbidden", "forbidden"], // whether guests can join space if other conditions are met
+ SPACE_JOIN_RULES: ["invite", "public", "public"],
+ ROOM_JOIN_RULES: ["restricted", "public", "public"]
+}
+
+const DEFAULT_PRIVACY_LEVEL = 0
+
/** @type {Map>} channel ID -> Promise */
const inflightRoomCreate = new Map()
@@ -69,7 +87,9 @@ function convertNameAndTopic(channel, guild, customName) {
*/
async function channelToKState(channel, guild) {
const spaceID = await createSpace.ensureSpace(guild)
- assert.ok(typeof spaceID === "string")
+ assert(typeof spaceID === "string")
+ const privacyLevel = select("guild_space", "privacy_level", {space_id: spaceID}).pluck().get()
+ assert(privacyLevel)
const row = select("channel_room", ["nick", "custom_avatar"], {channel_id: channel.id}).get()
const customName = row?.nick
@@ -84,27 +104,33 @@ async function channelToKState(channel, guild) {
avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API
}
- let history_visibility = "shared"
+ let history_visibility = PRIVACY_ENUMS.ROOM_HISTORY_VISIBILITY[privacyLevel]
if (channel["thread_metadata"]) history_visibility = "world_readable"
+ /** @type {{join_rule: string, allow?: any}} */
+ let join_rules = {
+ join_rule: "restricted",
+ allow: [{
+ type: "m.room_membership",
+ room_id: spaceID
+ }]
+ }
+ if (PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel] !== "restricted") {
+ join_rules = {join_rule: PRIVACY_ENUMS.ROOM_JOIN_RULES[privacyLevel]}
+ }
+
const channelKState = {
"m.room.name/": {name: convertedName},
"m.room.topic/": {topic: convertedTopic},
"m.room.avatar/": avatarEventContent,
- "m.room.guest_access/": {guest_access: "can_join"},
+ "m.room.guest_access/": {guest_access: PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
"m.room.history_visibility/": {history_visibility},
[`m.space.parent/${spaceID}`]: {
via: [reg.ooye.server_name],
canonical: true
},
/** @type {{join_rule: string, [x: string]: any}} */
- "m.room.join_rules/": {
- join_rule: "restricted",
- allow: [{
- type: "m.room_membership",
- room_id: spaceID
- }]
- },
+ "m.room.join_rules/": join_rules,
"m.room.power_levels/": {
events: {
"m.room.avatar": 0
@@ -132,7 +158,7 @@ async function channelToKState(channel, guild) {
}
}
- return {spaceID, channelKState}
+ return {spaceID, privacyLevel, channelKState}
}
/**
@@ -141,9 +167,10 @@ async function channelToKState(channel, guild) {
* @param guild
* @param {string} spaceID
* @param {any} kstate
+ * @param {number} privacyLevel
* @returns {Promise} room ID
*/
-async function createRoom(channel, guild, spaceID, kstate) {
+async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
let threadParent = null
if (channel.type === DiscordTypes.ChannelType.PublicThread) threadParent = channel.parent_id
@@ -160,8 +187,8 @@ async function createRoom(channel, guild, spaceID, kstate) {
const roomID = await api.createRoom({
name,
topic,
- preset: "private_chat", // This is closest to what we want, but properties from kstate override it anyway
- visibility: "private", // Not shown in the room directory
+ preset: PRIVACY_ENUMS.ROOM_HISTORY_VISIBILITY[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway
+ visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel],
invite: [],
initial_state: ks.kstateToState(kstate)
})
@@ -252,8 +279,8 @@ async function _syncRoom(channelID, shouldActuallySync) {
if (!existing) {
const creation = (async () => {
- const {spaceID, channelKState} = await channelToKState(channel, guild)
- const roomID = await createRoom(channel, guild, spaceID, channelKState)
+ const {spaceID, privacyLevel, channelKState} = await channelToKState(channel, guild)
+ const roomID = await createRoom(channel, guild, spaceID, channelKState, privacyLevel)
inflightRoomCreate.delete(channelID) // OK to release inflight waiters now. they will read the correct `existing` row
return roomID
})()
@@ -371,6 +398,8 @@ async function createAllForGuild(guildID) {
}
}
+module.exports.DEFAULT_PRIVACY_LEVEL = DEFAULT_PRIVACY_LEVEL
+module.exports.PRIVACY_ENUMS = PRIVACY_ENUMS
module.exports.createRoom = createRoom
module.exports.ensureRoom = ensureRoom
module.exports.syncRoom = syncRoom
diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js
index da877e0..1c1c357 100644
--- a/d2m/actions/create-space.js
+++ b/d2m/actions/create-space.js
@@ -32,8 +32,8 @@ async function createSpace(guild, kstate) {
const roomID = await createRoom.postApplyPowerLevels(kstate, async kstate => {
return api.createRoom({
name,
- preset: "private_chat", // cannot join space unless invited
- visibility: "private",
+ preset: createRoom.PRIVACY_ENUMS.PRESET[createRoom.DEFAULT_PRIVACY_LEVEL], // New spaces will have to use the default privacy level; we obviously can't look up the existing entry
+ visibility: createRoom.PRIVACY_ENUMS.VISIBILITY[createRoom.DEFAULT_PRIVACY_LEVEL],
power_level_content_override: {
events_default: 100, // space can only be managed by bridge
invite: 0 // any existing member can invite others
@@ -51,23 +51,21 @@ async function createSpace(guild, kstate) {
}
/**
- * @param {DiscordTypes.APIGuild} guild]
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {number} privacyLevel
*/
-async function guildToKState(guild) {
+async function guildToKState(guild, privacyLevel) {
const avatarEventContent = {}
if (guild.icon) {
avatarEventContent.discord_path = file.guildIcon(guild)
avatarEventContent.url = await file.uploadDiscordFileToMxc(avatarEventContent.discord_path) // TODO: somehow represent future values in kstate (callbacks?), while still allowing for diffing, so test cases don't need to touch the media API
}
- let history_visibility = "invited"
- if (guild["thread_metadata"]) history_visibility = "world_readable"
-
const guildKState = {
"m.room.name/": {name: guild.name},
"m.room.avatar/": avatarEventContent,
- "m.room.guest_access/": {guest_access: "can_join"}, // guests can join space if other conditions are met
- "m.room.history_visibility/": {history_visibility: "invited"} // any events sent after user was invited are visible
+ "m.room.guest_access/": {guest_access: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
+ "m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]}
}
return guildKState
@@ -86,11 +84,11 @@ async function _syncSpace(guild, shouldActuallySync) {
await inflightSpaceCreate.get(guild.id) // just waiting, and then doing a new db query afterwards, is the simplest way of doing it
}
- const spaceID = select("guild_space", "space_id", {guild_id: guild.id}).pluck().get()
+ const row = select("guild_space", ["space_id", "privacy_level"], {guild_id: guild.id}).get()
- if (!spaceID) {
+ if (!row) {
const creation = (async () => {
- const guildKState = await guildToKState(guild)
+ const guildKState = await guildToKState(guild, createRoom.DEFAULT_PRIVACY_LEVEL) // New spaces will have to use the default privacy level; we obviously can't look up the existing entry
const spaceID = await createSpace(guild, guildKState)
inflightSpaceCreate.delete(guild.id)
return spaceID
@@ -99,13 +97,15 @@ async function _syncSpace(guild, shouldActuallySync) {
return creation // Naturally, the newly created space is already up to date, so we can always skip syncing here.
}
+ const {space_id: spaceID, privacy_level} = row
+
if (!shouldActuallySync) {
return spaceID // only need to ensure space exists, and it does. return the space ID
}
console.log(`[space sync] to matrix: ${guild.name}`)
- const guildKState = await guildToKState(guild) // calling this in both branches because we don't want to calculate this if not syncing
+ const guildKState = await guildToKState(guild, privacy_level) // calling this in both branches because we don't want to calculate this if not syncing
// sync guild state to space
const spaceKState = await createRoom.roomToKState(spaceID)
@@ -159,17 +159,20 @@ async function syncSpaceFully(guildID) {
const guild = discord.guilds.get(guildID)
assert.ok(guild)
- const spaceID = select("guild_space", "space_id", {guild_id: guildID}).pluck().get()
+ const row = select("guild_space", ["space_id", "privacy_level"], {guild_id: guildID}).get()
- const guildKState = await guildToKState(guild)
-
- if (!spaceID) {
+ if (!row) {
+ const guildKState = await guildToKState(guild, createRoom.DEFAULT_PRIVACY_LEVEL)
const spaceID = await createSpace(guild, guildKState)
return spaceID // Naturally, the newly created space is already up to date, so we can always skip syncing here.
}
+ const {space_id: spaceID, privacy_level} = row
+
console.log(`[space sync] to matrix: ${guild.name}`)
+ const guildKState = await guildToKState(guild, privacy_level)
+
// sync guild state to space
const spaceKState = await createRoom.roomToKState(spaceID)
const spaceDiff = ks.diffKState(spaceKState, guildKState)
diff --git a/d2m/actions/send-message.js b/d2m/actions/send-message.js
index 843ee20..a0027b0 100644
--- a/d2m/actions/send-message.js
+++ b/d2m/actions/send-message.js
@@ -40,9 +40,11 @@ async function sendMessage(message, guild) {
}
for (const event of events) {
const eventType = event.$type
- /** @type {Pick> & { $type?: string }} */
+ if (event.$sender) senderMxid = event.$sender
+ /** @type {Pick> & { $type?: string, $sender?: string }} */
const eventWithoutType = {...event}
delete eventWithoutType.$type
+ delete eventWithoutType.$sender
const useTimestamp = message["backfill"] ? new Date(message.timestamp).getTime() : undefined
const eventID = await api.sendEvent(roomID, eventType, eventWithoutType, senderMxid, useTimestamp)
diff --git a/db/migrations/0006-add-privacy-to-space.sql b/db/migrations/0006-add-privacy-to-space.sql
new file mode 100644
index 0000000..a5a69e2
--- /dev/null
+++ b/db/migrations/0006-add-privacy-to-space.sql
@@ -0,0 +1,5 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE guild_space ADD COLUMN privacy_level TEXT NOT NULL DEFAULT 0;
+
+COMMIT;
diff --git a/db/orm-defs.d.ts b/db/orm-defs.d.ts
index 292a445..0714e0b 100644
--- a/db/orm-defs.d.ts
+++ b/db/orm-defs.d.ts
@@ -25,6 +25,7 @@ export type Models = {
guild_space: {
guild_id: string
space_id: string
+ privacy_level: number
}
lottie: {
diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql
index 1ce0467..2070b66 100644
--- a/test/ooye-test-data.sql
+++ b/test/ooye-test-data.sql
@@ -1,7 +1,7 @@
BEGIN TRANSACTION;
-INSERT INTO guild_space (guild_id, space_id) VALUES
-('112760669178241024', '!jjWAGMeQdNrVZSSfvz:cadence.moe');
+INSERT INTO guild_space (guild_id, space_id, privacy_level) VALUES
+('112760669178241024', '!jjWAGMeQdNrVZSSfvz:cadence.moe', 0);
INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom_avatar) VALUES
('112760669178241024', '!kLRqKKUQXcibIMtOpl:cadence.moe', 'heave', 'main', NULL, NULL),
From 42ba38e3dda88d44c2ec4ac2bcd543d65a98cc5d Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Thu, 12 Oct 2023 20:35:26 +1300
Subject: [PATCH 006/480] Use the correct property for guest access
---
d2m/actions/create-space.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js
index 1c1c357..59a5217 100644
--- a/d2m/actions/create-space.js
+++ b/d2m/actions/create-space.js
@@ -65,7 +65,7 @@ async function guildToKState(guild, privacyLevel) {
"m.room.name/": {name: guild.name},
"m.room.avatar/": avatarEventContent,
"m.room.guest_access/": {guest_access: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
- "m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]}
+ "m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.SPACE_HISTORY_VISIBILITY[privacyLevel]}
}
return guildKState
From 6b605b393d19daf5e1b56e08ab38c2eb30062d68 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Thu, 12 Oct 2023 20:47:33 +1300
Subject: [PATCH 007/480] Privacy level should be integer
---
db/migrations/0006-add-privacy-to-space.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/db/migrations/0006-add-privacy-to-space.sql b/db/migrations/0006-add-privacy-to-space.sql
index a5a69e2..ceb29e4 100644
--- a/db/migrations/0006-add-privacy-to-space.sql
+++ b/db/migrations/0006-add-privacy-to-space.sql
@@ -1,5 +1,5 @@
BEGIN TRANSACTION;
-ALTER TABLE guild_space ADD COLUMN privacy_level TEXT NOT NULL DEFAULT 0;
+ALTER TABLE guild_space ADD COLUMN privacy_level INTEGER NOT NULL DEFAULT 0;
COMMIT;
From 90579cea2843a14491977c82304ca091de991e63 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Thu, 12 Oct 2023 22:17:15 +1300
Subject: [PATCH 008/480] Only activate stdin reader when stdin is a TTY
---
stdin.js | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/stdin.js b/stdin.js
index c99a68b..587b176 100644
--- a/stdin.js
+++ b/stdin.js
@@ -21,15 +21,19 @@ const guildID = "112760669178241024"
const extraContext = {}
-setImmediate(() => { // assign after since old extraContext data will get removed
- if (!passthrough.repl) {
- const cli = repl.start({ prompt: "", eval: customEval, writer: s => s })
- Object.assign(cli.context, extraContext, passthrough)
- passthrough.repl = cli
- } else Object.assign(passthrough.repl.context, extraContext)
- // @ts-expect-error Says exit isn't assignable to a string
- sync.addTemporaryListener(passthrough.repl, "exit", () => process.exit())
-})
+if (process.stdin.isTTY) {
+ setImmediate(() => { // assign after since old extraContext data will get removed
+ if (!passthrough.repl) {
+ const cli = repl.start({ prompt: "", eval: customEval, writer: s => s })
+ Object.assign(cli.context, extraContext, passthrough)
+ passthrough.repl = cli
+ } else {
+ Object.assign(passthrough.repl.context, extraContext)
+ }
+ // @ts-expect-error Says exit isn't assignable to a string
+ sync.addTemporaryListener(passthrough.repl, "exit", () => process.exit())
+ })
+}
/**
* @param {string} input
From 98f5aeb45a6cb6c3e413ce3d3b6d0f4fef375fd9 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Thu, 12 Oct 2023 22:32:28 +1300
Subject: [PATCH 009/480] Potentially fix privacy level assertion error
---
d2m/actions/create-room.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js
index bee45e2..d1e1416 100644
--- a/d2m/actions/create-room.js
+++ b/d2m/actions/create-room.js
@@ -89,7 +89,7 @@ async function channelToKState(channel, guild) {
const spaceID = await createSpace.ensureSpace(guild)
assert(typeof spaceID === "string")
const privacyLevel = select("guild_space", "privacy_level", {space_id: spaceID}).pluck().get()
- assert(privacyLevel)
+ assert(typeof privacyLevel === "number")
const row = select("channel_room", ["nick", "custom_avatar"], {channel_id: channel.id}).get()
const customName = row?.nick
From 1b03a39dee7ac30a847796e6d7860d6fc84b1fc2 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Thu, 12 Oct 2023 22:34:23 +1300
Subject: [PATCH 010/480] put the preset in the preset field......
---
d2m/actions/create-room.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js
index d1e1416..e9e2b3e 100644
--- a/d2m/actions/create-room.js
+++ b/d2m/actions/create-room.js
@@ -187,7 +187,7 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
const roomID = await api.createRoom({
name,
topic,
- preset: PRIVACY_ENUMS.ROOM_HISTORY_VISIBILITY[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway
+ preset: PRIVACY_ENUMS.PRESET[privacyLevel], // This is closest to what we want, but properties from kstate override it anyway
visibility: PRIVACY_ENUMS.VISIBILITY[privacyLevel],
invite: [],
initial_state: ks.kstateToState(kstate)
From 3f3851d1f506fa0bef09f3e6a6562fc3d57d7862 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Thu, 12 Oct 2023 22:45:27 +1300
Subject: [PATCH 011/480] Also sync join rules for space
---
d2m/actions/create-space.js | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js
index 59a5217..9e55794 100644
--- a/d2m/actions/create-space.js
+++ b/d2m/actions/create-space.js
@@ -65,7 +65,8 @@ async function guildToKState(guild, privacyLevel) {
"m.room.name/": {name: guild.name},
"m.room.avatar/": avatarEventContent,
"m.room.guest_access/": {guest_access: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
- "m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.SPACE_HISTORY_VISIBILITY[privacyLevel]}
+ "m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.SPACE_HISTORY_VISIBILITY[privacyLevel]},
+ "m.room.join_rules/": {join_rule: createRoom.PRIVACY_ENUMS.SPACE_JOIN_RULES[privacyLevel]}
}
return guildKState
From e84e0d28db7cdaf1e89676b202eb7b472795960e Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Thu, 12 Oct 2023 22:55:52 +1300
Subject: [PATCH 012/480] invited users in registration should be admin
---
d2m/actions/create-room.js | 3 ++-
d2m/actions/create-space.js | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/d2m/actions/create-room.js b/d2m/actions/create-room.js
index e9e2b3e..6994b57 100644
--- a/d2m/actions/create-room.js
+++ b/d2m/actions/create-room.js
@@ -134,7 +134,8 @@ async function channelToKState(channel, guild) {
"m.room.power_levels/": {
events: {
"m.room.avatar": 0
- }
+ },
+ users: reg.ooye.invite.reduce((a, c) => (a[c] = 100, a), {})
},
"chat.schildi.hide_ui/read_receipts": {
hidden: true
diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js
index 9e55794..9cb0c41 100644
--- a/d2m/actions/create-space.js
+++ b/d2m/actions/create-space.js
@@ -66,7 +66,8 @@ async function guildToKState(guild, privacyLevel) {
"m.room.avatar/": avatarEventContent,
"m.room.guest_access/": {guest_access: createRoom.PRIVACY_ENUMS.GUEST_ACCESS[privacyLevel]},
"m.room.history_visibility/": {history_visibility: createRoom.PRIVACY_ENUMS.SPACE_HISTORY_VISIBILITY[privacyLevel]},
- "m.room.join_rules/": {join_rule: createRoom.PRIVACY_ENUMS.SPACE_JOIN_RULES[privacyLevel]}
+ "m.room.join_rules/": {join_rule: createRoom.PRIVACY_ENUMS.SPACE_JOIN_RULES[privacyLevel]},
+ "m.room.power_levels/": {users: reg.ooye.invite.reduce((a, c) => (a[c] = 100, a), {})}
}
return guildKState
From 3a6b40e27d55838d7405c1675004b55deac2a9e6 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Fri, 13 Oct 2023 00:37:18 +1300
Subject: [PATCH 013/480] Add Discord command for setting privacy level
---
d2m/actions/create-space.js | 2 +-
discord/discord-command-handler.js | 35 ++++++++++++++++++++++++++++++
2 files changed, 36 insertions(+), 1 deletion(-)
diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js
index 9cb0c41..b30f681 100644
--- a/d2m/actions/create-space.js
+++ b/d2m/actions/create-space.js
@@ -154,7 +154,7 @@ function syncSpace(guild) {
/**
* Inefficiently force the space and its existing child rooms to be fully updated.
- * Should not need to be called as part of the bridge's normal operation.
+ * Prefer not to call this as part of the bridge's normal operation.
*/
async function syncSpaceFully(guildID) {
/** @ts-ignore @type {DiscordTypes.APIGuild} */
diff --git a/discord/discord-command-handler.js b/discord/discord-command-handler.js
index 5711561..f9f07c0 100644
--- a/discord/discord-command-handler.js
+++ b/discord/discord-command-handler.js
@@ -11,6 +11,8 @@ const {discord, sync, db, select} = require("../passthrough")
const api = sync.require("../matrix/api")
/** @type {import("../matrix/file")} */
const file = sync.require("../matrix/file")
+/** @type {import("../d2m/actions/create-space")} */
+const createSpace = sync.require("../d2m/actions/create-space")
/** @type {import("./utils")} */
const utils = sync.require("./utils")
@@ -212,6 +214,39 @@ const commands = [{
})
}
)
+}, {
+ aliases: ["privacy", "discoverable", "publish", "published"],
+ execute: replyctx(
+ async (message, channel, guild, ctx) => {
+ const current = select("guild_space", "privacy_level", {guild_id: guild.id}).pluck().get()
+ if (current == null) {
+ return discord.snow.channel.createMessage(channel.id, {
+ ...ctx,
+ content: "This server isn't bridged to the other side."
+ })
+ }
+
+ const levels = ["invite", "link", "directory"]
+ const level = levels.findIndex(x => message.content.includes(x))
+ if (level === -1) {
+ return discord.snow.channel.createMessage(channel.id, {
+ ...ctx,
+ content: "**Usage: `//privacy `**. This will set who can join the space on Matrix-side. There are three levels:"
+ + "\n`invite`: Can only join with a direct in-app invite from another Matrix user, or the //invite command."
+ + "\n`link`: Matrix links can be created and shared like Discord's invite links. `invite` features also work."
+ + "\n`directory`: Publishes to the Matrix in-app directory, like Server Discovery. Preview enabled. `invite` and `link` also work."
+ + `\n**Current privacy level: \`${levels[current]}\`**`
+ })
+ }
+
+ db.prepare("UPDATE guild_space SET privacy_level = ? WHERE guild_id = ?").run(level, guild.id)
+ discord.snow.channel.createMessage(channel.id, {
+ ...ctx,
+ content: `Privacy level updated to \`${levels[level]}\`. Changes will apply shortly.`
+ })
+ await createSpace.syncSpaceFully(guild.id)
+ }
+ )
}]
/** @type {CommandExecute} */
From a1710af542cfbd75748304c006d41a30c6b461fe Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Fri, 13 Oct 2023 00:39:45 +1300
Subject: [PATCH 014/480] Explain that auto-invite also marks as admin
---
registration.example.yaml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/registration.example.yaml b/registration.example.yaml
index bad42fe..9e7cd2c 100644
--- a/registration.example.yaml
+++ b/registration.example.yaml
@@ -19,4 +19,5 @@ ooye:
server_name: [the part after the colon in your matrix id, like cadence.moe]
server_origin: [the full protocol and domain of your actual matrix server's location, with no trailing slash, like https://matrix.cadence.moe]
invite:
- # - @cadence:cadence.moe # uncomment this to auto-invite the named user to newly created spaces
+ # uncomment this to auto-invite the named user to newly created spaces and mark them as admin (PL 100) everywhere
+ # - @cadence:cadence.moe
From 1ad1c6b525c924d29b1e17e638547601427b4322 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Fri, 13 Oct 2023 00:51:38 +1300
Subject: [PATCH 015/480] Add tests for all room privacy levels
---
d2m/actions/create-room.test.js | 29 ++++++++++++++++++++++++++++-
test/data.js | 3 +++
test/test.js | 1 +
3 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/d2m/actions/create-room.test.js b/d2m/actions/create-room.test.js
index e40bf6f..93f9203 100644
--- a/d2m/actions/create-room.test.js
+++ b/d2m/actions/create-room.test.js
@@ -5,7 +5,34 @@ const {kstateStripConditionals} = require("../../matrix/kstate")
const {test} = require("supertape")
const testData = require("../../test/data")
-test("channel2room: general", async t => {
+const passthrough = require("../../passthrough")
+const {db} = passthrough
+
+test("channel2room: discoverable privacy room", async t => {
+ db.prepare("UPDATE guild_space SET privacy_level = 2").run()
+ t.deepEqual(
+ kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)),
+ Object.assign({}, testData.room.general, {
+ "m.room.guest_access/": {guest_access: "forbidden"},
+ "m.room.join_rules/": {join_rule: "public"},
+ "m.room.history_visibility/": {history_visibility: "world_readable"}
+ })
+ )
+})
+
+test("channel2room: linkable privacy room", async t => {
+ db.prepare("UPDATE guild_space SET privacy_level = 1").run()
+ t.deepEqual(
+ kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)),
+ Object.assign({}, testData.room.general, {
+ "m.room.guest_access/": {guest_access: "forbidden"},
+ "m.room.join_rules/": {join_rule: "public"}
+ })
+ )
+})
+
+test("channel2room: invite-only privacy room", async t => {
+ db.prepare("UPDATE guild_space SET privacy_level = 0").run()
t.deepEqual(
kstateStripConditionals(await channelToKState(testData.channel.general, testData.guild.general).then(x => x.channelKState)),
testData.room.general
diff --git a/test/data.js b/test/data.js
index 4919beb..c5a1f8e 100644
--- a/test/data.js
+++ b/test/data.js
@@ -44,6 +44,9 @@ module.exports = {
"m.room.power_levels/": {
events: {
"m.room.avatar": 0
+ },
+ users: {
+ "@test_auto_invite:example.org": 100
}
},
"chat.schildi.hide_ui/read_receipts": {hidden: true},
diff --git a/test/test.js b/test/test.js
index 280503d..5e15a14 100644
--- a/test/test.js
+++ b/test/test.js
@@ -14,6 +14,7 @@ const db = new sqlite(":memory:")
const reg = require("../matrix/read-registration")
reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded
+reg.ooye.invite = ["@test_auto_invite:example.org"]
const sync = new HeatSync({watchFS: false})
From 93fa5b2e9a851df3cb45699c43ca79137d51fa15 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Fri, 13 Oct 2023 01:07:52 +1300
Subject: [PATCH 016/480] Require permissions to change privacy
---
discord/discord-command-handler.js | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/discord/discord-command-handler.js b/discord/discord-command-handler.js
index f9f07c0..aad06e1 100644
--- a/discord/discord-command-handler.js
+++ b/discord/discord-command-handler.js
@@ -239,6 +239,15 @@ const commands = [{
})
}
+ assert(message.member)
+ const guildPermissions = utils.getPermissions(message.member.roles, guild.roles)
+ if (guild.owner_id !== message.author.id && !(guildPermissions & BigInt(0x28))) { // MANAGE_GUILD | ADMINISTRATOR
+ return discord.snow.channel.createMessage(channel.id, {
+ ...ctx,
+ content: "You don't have permission to change the privacy level. You need Manage Server or Administrator."
+ })
+ }
+
db.prepare("UPDATE guild_space SET privacy_level = ? WHERE guild_id = ?").run(level, guild.id)
discord.snow.channel.createMessage(channel.id, {
...ctx,
From 61ac53599535b43da242ec7b23a31c80c09072f6 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Fri, 13 Oct 2023 02:05:44 +1300
Subject: [PATCH 017/480] d->m fix replying to matrix user in thread
---
d2m/converters/message-to-event.js | 5 ++
d2m/converters/message-to-event.test.js | 39 +++++++++++
m2d/actions/send-event.js | 2 +-
test/data.js | 86 +++++++++++++++++++++++++
test/ooye-test-data.sql | 7 +-
5 files changed, 136 insertions(+), 3 deletions(-)
diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js
index 09ac39a..7c4a998 100644
--- a/d2m/converters/message-to-event.js
+++ b/d2m/converters/message-to-event.js
@@ -215,6 +215,11 @@ async function messageToEvent(message, guild, options = {}, di) {
repliedToUserHtml = repliedToDisplayName
}
let repliedToContent = message.referenced_message?.content
+ if (repliedToContent?.startsWith("> <:L1:")) {
+ // If the Discord user is replying to a Matrix user's reply, the fallback is going to contain the emojis and stuff from the bridged rep of the Matrix user's reply quote.
+ // Need to remove that previous reply rep from this fallback body. The fallbody body should only contain the Matrix user's actual message.
+ repliedToContent = repliedToContent.split("\n").slice(2).join("\n")
+ }
if (repliedToContent == "") repliedToContent = "[Media]"
else if (!repliedToContent) repliedToContent = "[Replied-to message content wasn't provided by Discord]"
const repliedToHtml = markdown.toHTML(repliedToContent, {
diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js
index 19f0111..4b4b6a0 100644
--- a/d2m/converters/message-to-event.test.js
+++ b/d2m/converters/message-to-event.test.js
@@ -273,6 +273,45 @@ test("message2event: simple reply to matrix user, reply fallbacks disabled", asy
}])
})
+test("message2event: simple reply in thread to a matrix user's reply", async t => {
+ const events = await messageToEvent(data.message.simple_reply_to_reply_in_thread, data.guild.general, {}, {
+ api: {
+ getEvent: mockGetEvent(t, "!FuDZhlOAtqswlyxzeR:cadence.moe", "$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo", {
+ type: "m.room.message",
+ sender: "@cadence:cadence.moe",
+ content: {
+ msgtype: "m.text",
+ body: "> <@_ooye_cadence:cadence.moe> So what I'm wondering is about replies.\n\nWhat about them?",
+ format: "org.matrix.custom.html",
+ formatted_body: "In reply to @_ooye_cadence:cadence.moe
So what I'm wondering is about replies.
What about them?",
+ "m.relates_to": {
+ "m.in_reply_to": {
+ event_id: "$fWQT8uOrzLzAXNVXz88VkGx7Oo724iS5uD8Qn5KUy9w"
+ }
+ }
+ },
+ event_id: "$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo",
+ room_id: "!FuDZhlOAtqswlyxzeR:cadence.moe"
+ })
+ }
+ })
+ t.deepEqual(events, [{
+ $type: "m.room.message",
+ "m.relates_to": {
+ "m.in_reply_to": {
+ event_id: "$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo"
+ }
+ },
+ "m.mentions": {
+ user_ids: ["@cadence:cadence.moe"]
+ },
+ msgtype: "m.text",
+ body: "> cadence: What about them?\n\nWell, they don't seem to...",
+ format: "org.matrix.custom.html",
+ formatted_body: "In reply to cadence
What about them?
Well, they don't seem to...",
+ }])
+})
+
test("message2event: simple written @mention for matrix user", async t => {
const events = await messageToEvent(data.message.simple_written_at_mention_for_matrix, data.guild.general, {}, {
api: {
diff --git a/m2d/actions/send-event.js b/m2d/actions/send-event.js
index 13966fb..c1e3ba3 100644
--- a/m2d/actions/send-event.js
+++ b/m2d/actions/send-event.js
@@ -99,7 +99,7 @@ async function sendEvent(event) {
for (const message of messagesToSend) {
const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message, threadID)
- db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(messageResponse.id, channelID)
+ db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(messageResponse.id, threadID || channelID)
db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content["msgtype"] || null, messageResponse.id, eventPart) // source 0 = matrix
eventPart = 1
diff --git a/test/data.js b/test/data.js
index c5a1f8e..ea9f66a 100644
--- a/test/data.js
+++ b/test/data.js
@@ -939,6 +939,92 @@ module.exports = {
attachments: [],
guild_id: "112760669178241024"
},
+ simple_reply_to_reply_in_thread: {
+ type: 19,
+ tts: false,
+ timestamp: "2023-10-12T12:35:12.721000+00:00",
+ referenced_message: {
+ webhook_id: "1142275246532083723",
+ type: 0,
+ tts: false,
+ timestamp: "2023-10-12T12:35:06.578000+00:00",
+ position: 1,
+ pinned: false,
+ mentions: [
+ {
+ username: "cadence.worm",
+ public_flags: 0,
+ id: "772659086046658620",
+ global_name: "cadence",
+ discriminator: "0",
+ avatar_decoration_data: null,
+ avatar: "4b5c4b28051144e4c111f0113a0f1cf1"
+ }
+ ],
+ mention_roles: [],
+ mention_everyone: false,
+ id: "1162005526675193909",
+ flags: 0,
+ embeds: [],
+ edited_timestamp: null,
+ content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/1100319549670301727/1162005314908999790/1162005501782011975 <@772659086046658620>:\n" +
+ "> So what I'm wondering is about replies.\n" +
+ "What about them?",
+ components: [],
+ channel_id: "1162005314908999790",
+ author: {
+ username: "cadence [they]",
+ id: "1142275246532083723",
+ global_name: null,
+ discriminator: "0000",
+ bot: true,
+ avatar: "af0ead3b92cf6e448fdad80b4e7fc9e5"
+ },
+ attachments: [],
+ application_id: "684280192553844747"
+ },
+ position: 2,
+ pinned: false,
+ nonce: "1162005551190638592",
+ message_reference: {
+ message_id: "1162005526675193909",
+ guild_id: "1100319549670301727",
+ channel_id: "1162005314908999790"
+ },
+ mentions: [],
+ mention_roles: [],
+ mention_everyone: false,
+ member: {
+ roles: [],
+ premium_since: null,
+ pending: false,
+ nick: "worm",
+ mute: false,
+ joined_at: "2023-04-25T07:17:03.696000+00:00",
+ flags: 0,
+ deaf: false,
+ communication_disabled_until: null,
+ avatar: null
+ },
+ id: "1162005552440815646",
+ flags: 0,
+ embeds: [],
+ edited_timestamp: null,
+ content: "Well, they don't seem to...",
+ components: [],
+ channel_id: "1162005314908999790",
+ author: {
+ username: "cadence.worm",
+ public_flags: 0,
+ id: "772659086046658620",
+ global_name: "cadence",
+ discriminator: "0",
+ avatar_decoration_data: null,
+ avatar: "4b5c4b28051144e4c111f0113a0f1cf1"
+ },
+ attachments: [],
+ guild_id: "1100319549670301727"
+ },
sticker: {
id: "1106366167788044450",
type: 0,
diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql
index 2070b66..f5cfb5c 100644
--- a/test/ooye-test-data.sql
+++ b/test/ooye-test-data.sql
@@ -8,6 +8,7 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom
('497161350934560778', '!CzvdIdUQXgUjDVKxeU:cadence.moe', 'amanda-spam', NULL, NULL, NULL),
('160197704226439168', '!hYnGGlPHlbujVVfktC:cadence.moe', 'the-stanley-parable-channel', 'bots', NULL, NULL),
('1100319550446252084', '!BnKuBPCvyfOkhcUjEu:cadence.moe', 'worm-farm', NULL, NULL, NULL),
+('1162005314908999790', '!FuDZhlOAtqswlyxzeR:cadence.moe', 'Hey.', NULL, '1100319550446252084', NULL),
('297272183716052993', '!rEOspnYqdOalaIFniV:cadence.moe', 'general', NULL, NULL, NULL),
('122155380120748034', '!cqeGDbPiMFAhLsqqqq:cadence.moe', 'cadences-mind', 'coding', NULL, NULL);
@@ -36,7 +37,8 @@ INSERT INTO message_channel (message_id, channel_id) VALUES
('1144865310588014633', '687028734322147344'),
('1145688633186193479', '1100319550446252084'),
('1145688633186193480', '1100319550446252084'),
-('1145688633186193481', '1100319550446252084');
+('1145688633186193481', '1100319550446252084'),
+('1162005526675193909', '1162005314908999790');
INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES
('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 1),
@@ -57,7 +59,8 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part
('$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8', 'm.room.message', 'm.text', '1144874214311067708', 0, 0),
('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs', 'm.room.message', 'm.text', '1145688633186193479', 0, 0),
('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193480', 0, 0),
-('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193481', 1, 0);
+('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193481', 1, 0),
+('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0);
INSERT INTO file (discord_url, mxc_url) VALUES
('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'),
From 850de85d8250a00b5a2eb744163bf64a7e9c9a77 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Fri, 13 Oct 2023 02:06:06 +1300
Subject: [PATCH 018/480] Add //thread command for Matrix users
---
docs/user-guide.md | 2 ++
matrix/matrix-command-handler.js | 41 ++++++++++++++++++++++++++++----
2 files changed, 38 insertions(+), 5 deletions(-)
diff --git a/docs/user-guide.md b/docs/user-guide.md
index 0228e87..d360806 100644
--- a/docs/user-guide.md
+++ b/docs/user-guide.md
@@ -28,6 +28,8 @@ If a room is newly created, it will be added to the space, but it will not be an
If a thread is newly created, it will be added to the space, and an announcement will also be posted to the parent channel with a link to quickly join.
+Matrix users can create their own thread with `/thread `. This will create a real thread channel on Discord-side and announce its creation on both sides in the usual way.
+
## Custom Room Icons
Normally on Matrix, the room icons will match the space icon. Since Matrix allows for room-specific icons, the bridge will keep track of any custom icon that was set on a room.
diff --git a/matrix/matrix-command-handler.js b/matrix/matrix-command-handler.js
index a22d2b1..8056f96 100644
--- a/matrix/matrix-command-handler.js
+++ b/matrix/matrix-command-handler.js
@@ -73,6 +73,7 @@ function onReactionAdd(event) {
* @callback CommandExecute
* @param {Ty.Event.Outer_M_Room_Message} event
* @param {string} realBody
+ * @param {string[]} words
* @param {any} [ctx]
*/
@@ -85,13 +86,13 @@ function onReactionAdd(event) {
/** @param {CommandExecute} execute */
function replyctx(execute) {
/** @type {CommandExecute} */
- return function(event, realBody, ctx = {}) {
+ return function(event, realBody, words, ctx = {}) {
ctx["m.relates_to"] = {
"m.in_reply_to": {
event_id: event.event_id
}
}
- return execute(event, realBody, ctx)
+ return execute(event, realBody, words, ctx)
}
}
@@ -148,7 +149,7 @@ class MatrixStringBuilder {
const commands = [{
aliases: ["emoji"],
execute: replyctx(
- async (event, realBody, ctx) => {
+ async (event, realBody, words, ctx) => {
// Guard
/** @type {string} */ // @ts-ignore
const channelID = select("channel_room", "channel_id", {room_id: event.room_id}).pluck().get()
@@ -165,7 +166,7 @@ const commands = [{
const permissions = dUtils.getPermissions([], guild.roles)
if (guild.emojis.length >= slots) {
matrixOnlyReason = "CAPACITY"
- } else if (!(permissions | 0x40000000n)) { // MANAGE_GUILD_EXPRESSIONS (apparently CREATE_GUILD_EXPRESSIONS isn't good enough...)
+ } else if (!(permissions & 0x40000000n)) { // MANAGE_GUILD_EXPRESSIONS (apparently CREATE_GUILD_EXPRESSIONS isn't good enough...)
matrixOnlyReason = "USER_PERMISSIONS"
}
}
@@ -284,6 +285,36 @@ const commands = [{
})
}
)
+}, {
+ aliases: ["thread"],
+ execute: replyctx(
+ async (event, realBody, words, ctx) => {
+ // Guard
+ /** @type {string} */ // @ts-ignore
+ const channelID = select("channel_room", "channel_id", {room_id: event.room_id}).pluck().get()
+ const guildID = discord.channels.get(channelID)?.["guild_id"]
+ if (!guildID) {
+ return api.sendEvent(event.room_id, "m.room.message", {
+ ...ctx,
+ msgtype: "m.text",
+ body: "This room isn't bridged to the other side."
+ })
+ }
+
+ const guild = discord.guilds.get(guildID)
+ assert(guild)
+ const permissions = dUtils.getPermissions([], guild.roles)
+ if (!(permissions & 0x800000000n)) { // CREATE_PUBLIC_THREADS
+ return api.sendEvent(event.room_id, "m.room.message", {
+ ...ctx,
+ msgtype: "m.text",
+ body: "This command creates a thread on Discord. But you aren't allowed to do this, because if you were a Discord user, you wouldn't have the Create Public Threads permission."
+ })
+ }
+
+ await discord.snow.channel.createThreadWithoutMessage(channelID, {type: 11, name: words.slice(1).join(" ")})
+ }
+ )
}]
@@ -308,7 +339,7 @@ async function execute(event) {
const command = commands.find(c => c.aliases.includes(commandName))
if (!command) return
- await command.execute(event, realBody)
+ await command.execute(event, realBody, words)
}
module.exports.execute = execute
From 67305bb636ad2ba5f58fdba485e71b85a58317e2 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Fri, 13 Oct 2023 21:32:42 +1300
Subject: [PATCH 019/480] Allow m.notice (embeds) to be edited
---
d2m/converters/edit-to-changes.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/d2m/converters/edit-to-changes.js b/d2m/converters/edit-to-changes.js
index 9d1fc0b..dc42708 100644
--- a/d2m/converters/edit-to-changes.js
+++ b/d2m/converters/edit-to-changes.js
@@ -102,7 +102,7 @@ async function editToChanges(message, guild, api) {
// So we'll remove entries from eventsToReplace that *definitely* cannot have changed. (This is category 4 mentioned above.) Everything remaining *may* have changed.
eventsToReplace = eventsToReplace.filter(ev => {
// Discord does not allow files, images, attachments, or videos to be edited.
- if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote") {
+ if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote" && ev.old.event_subtype !== "m.notice") {
return false
}
// Discord does not allow stickers to be edited.
From 480c7a6bd9c8fb82f545bf8b82d2d3f735f09a78 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Fri, 13 Oct 2023 23:05:07 +1300
Subject: [PATCH 020/480] m->d: Use Matrix displayname in m/m reply preview
---
m2d/converters/event-to-message.js | 3 ++-
m2d/converters/event-to-message.test.js | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js
index fcf0bf8..ff31607 100644
--- a/m2d/converters/event-to-message.js
+++ b/m2d/converters/event-to-message.js
@@ -324,11 +324,12 @@ async function eventToMessage(event, guild, di) {
replyLine += `https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} `
}
const sender = repliedToEvent.sender
- const senderName = sender.match(/@([^:]*)/)?.[1] || sender
const authorID = select("sim", "user_id", {mxid: repliedToEvent.sender}).pluck().get()
if (authorID) {
replyLine += `<@${authorID}>`
} else {
+ let senderName = select("member_cache", "displayname", {mxid: repliedToEvent.sender}).pluck().get()
+ if (!senderName) senderName = sender.match(/@([^:]*)/)?.[1] || sender
replyLine += `Ⓜ️**${senderName}**`
}
// If the event has been edited, the homeserver will include the relation in `unsigned`.
diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js
index 87a4b12..41d63b2 100644
--- a/m2d/converters/event-to-message.test.js
+++ b/m2d/converters/event-to-message.test.js
@@ -1151,7 +1151,7 @@ test("event2message: rich reply to a matrix user's long message with formatting"
messagesToEdit: [],
messagesToSend: [{
username: "cadence [they]",
- content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence**:"
+ content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence [they]**:"
+ "\n> i should have a little happy test list bold em..."
+ "\n**no you can't!!!**",
avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU"
@@ -1312,7 +1312,7 @@ test("event2message: with layered rich replies, the preview should only be the r
messagesToEdit: [],
messagesToSend: [{
username: "cadence [they]",
- content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence**:"
+ content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/687028734322147344/1144865310588014633 Ⓜ️**cadence [they]**:"
+ "\n> two"
+ "\nthree",
avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU"
From 999276e4073ca0d941f4e7f936ececf94aa907da Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Fri, 13 Oct 2023 23:23:15 +1300
Subject: [PATCH 021/480] m->d: Fix HTML entities showing in reply preview
---
m2d/converters/event-to-message.js | 13 ++++---
m2d/converters/event-to-message.test.js | 49 +++++++++++++++++++++++++
package-lock.json | 12 ++++++
package.json | 1 +
readme.md | 1 +
5 files changed, 71 insertions(+), 5 deletions(-)
diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js
index ff31607..e7bbda5 100644
--- a/m2d/converters/event-to-message.js
+++ b/m2d/converters/event-to-message.js
@@ -5,6 +5,7 @@ const DiscordTypes = require("discord-api-types/v10")
const chunk = require("chunk-text")
const TurndownService = require("turndown")
const assert = require("assert").strict
+const entities = require("entities")
const passthrough = require("../../passthrough")
const {sync, db, discord, select, from} = passthrough
@@ -349,11 +350,13 @@ async function eventToMessage(event, guild, di) {
} else {
const repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body
const contentPreviewChunks = chunk(
- repliedToContent.replace(/.*<\/mx-reply>/, "") // Remove everything before replies, so just use the actual message body
- .replace(/.*?<\/blockquote>/, "") // If the message starts with a blockquote, don't count it and use the message body afterwards
- .replace(/(?:\n|
)+/g, " ") // Should all be on one line
- .replace(/]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
- .replace(/<[^>]+>/g, ""), 50) // Completely strip all other formatting.
+ entities.decodeHTML5Strict( // Remove entities like & "
+ repliedToContent.replace(/.*<\/mx-reply>/, "") // Remove everything before replies, so just use the actual message body
+ .replace(/.*?<\/blockquote>/, "") // If the message starts with a blockquote, don't count it and use the message body afterwards
+ .replace(/(?:\n|
)+/g, " ") // Should all be on one line
+ .replace(/]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
+ .replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting.
+ ), 50)
contentPreview = ":\n> "
contentPreview += contentPreviewChunks.length > 1 ? contentPreviewChunks[0] + "..." : contentPreviewChunks[0]
}
diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js
index 41d63b2..1a1c3f0 100644
--- a/m2d/converters/event-to-message.test.js
+++ b/m2d/converters/event-to-message.test.js
@@ -813,6 +813,55 @@ test("event2message: should include a reply preview when message ends with a blo
)
})
+test("event2message: entities are not escaped in main message or reply preview", async t => {
+ // Intended result: Testing? in italics, followed by the sequence "':.`[]&things
+ t.deepEqual(
+ await eventToMessage({
+ type: "m.room.message",
+ sender: "@cadence:cadence.moe",
+ content: {
+ msgtype: "m.text",
+ body: "> <@cadence:cadence.moe> _Testing?_ \"':.`[]&things\n\n_Testing?_ \"':.`[]&things",
+ format: "org.matrix.custom.html",
+ formatted_body: "In reply to @cadence:cadence.moe
Testing? \"':.`[]&things
Testing? "':.`[]&things",
+ "m.relates_to": {
+ "m.in_reply_to": {
+ event_id: "$yIWjZPi6Xk56fBxJwqV4ANs_hYLjnWI2cNKbZ2zwk60"
+ }
+ }
+ },
+ event_id: "$2I7odT9okTdpwDcqOjkJb_A3utdO4V8Cp3LK6-Rvwcs",
+ room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe"
+ }, data.guild.general, {
+ api: {
+ getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$yIWjZPi6Xk56fBxJwqV4ANs_hYLjnWI2cNKbZ2zwk60", {
+ type: "m.room.message",
+ sender: "@cadence:cadence.moe",
+ content: {
+ "msgtype": "m.text",
+ "body": "_Testing?_ \"':.`[]&things",
+ "format": "org.matrix.custom.html",
+ "formatted_body": "Testing? "':.`[]&things"
+ },
+ event_id: "$yIWjZPi6Xk56fBxJwqV4ANs_hYLjnWI2cNKbZ2zwk60",
+ room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe"
+ })
+ }
+ }),
+ {
+ messagesToDelete: [],
+ messagesToEdit: [],
+ messagesToSend: [{
+ username: "cadence [they]",
+ content: "> <:L1:1144820033948762203><:L2:1144820084079087647>Ⓜ️**cadence [they]**:"
+ + "\n> Testing? \"':.`[]&things"
+ + "\n_Testing?_ \"':.\\`\\[\\]&things",
+ avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU"
+ }]
+ }
+ )
+})
+
test("event2message: editing a rich reply to a sim user", async t => {
const eventsFetched = []
t.deepEqual(
diff --git a/package-lock.json b/package-lock.json
index d1ed09e..3847ee1 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -14,6 +14,7 @@
"chunk-text": "^2.0.1",
"cloudstorm": "^0.8.0",
"discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3",
+ "entities": "^4.5.0",
"giframe": "github:cloudrac3r/giframe#v0.4.1",
"heatsync": "^2.4.1",
"js-yaml": "^4.1.0",
@@ -1096,6 +1097,17 @@
"once": "^1.4.0"
}
},
+ "node_modules/entities": {
+ "version": "4.5.0",
+ "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+ "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+ "engines": {
+ "node": ">=0.12"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/entities?sponsor=1"
+ }
+ },
"node_modules/es-get-iterator": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz",
diff --git a/package.json b/package.json
index e8eec83..6a9deea 100644
--- a/package.json
+++ b/package.json
@@ -20,6 +20,7 @@
"chunk-text": "^2.0.1",
"cloudstorm": "^0.8.0",
"discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3",
+ "entities": "^4.5.0",
"giframe": "github:cloudrac3r/giframe#v0.4.1",
"heatsync": "^2.4.1",
"js-yaml": "^4.1.0",
diff --git a/readme.md b/readme.md
index 77239f7..5c97745 100644
--- a/readme.md
+++ b/readme.md
@@ -164,6 +164,7 @@ Follow these steps:
* (1) discord-markdown: This is my fork! I make sure it does what I want.
* (0) giframe: This is my fork! It should do what I want.
* (1) heatsync: Module hot-reloader that I trust.
+* (0) entities: Looks fine. No dependencies.
* (1) js-yaml: It seems to do what I want, and it's already pulled in by matrix-appservice.
* (70) matrix-appservice: I wish it didn't pull in express :(
* (0) minimist: It's already pulled in by better-sqlite3->prebuild-install
From 44f90cbb5fd2780636f8f44937adb65c7fd0cfb2 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Sat, 14 Oct 2023 01:00:28 +1300
Subject: [PATCH 022/480] Use Discord global_name as sim user displayname
---
d2m/actions/register-user.js | 2 +-
d2m/actions/register-user.test.js | 20 ++++++++++++++++++
test/data.js | 34 +++++++++++++++++++++++++++++++
test/ooye-test-data.sql | 3 ++-
4 files changed, 57 insertions(+), 2 deletions(-)
diff --git a/d2m/actions/register-user.js b/d2m/actions/register-user.js
index 62e0fb6..9b5527f 100644
--- a/d2m/actions/register-user.js
+++ b/d2m/actions/register-user.js
@@ -97,7 +97,7 @@ async function ensureSimJoined(user, roomID) {
*/
async function memberToStateContent(user, member, guildID) {
let displayname = user.username
- // if (member.nick && member.nick !== displayname) displayname = member.nick + " | " + displayname // prepend nick if present
+ if (user.global_name) displayname = user.global_name
if (member.nick) displayname = member.nick
const content = {
diff --git a/d2m/actions/register-user.test.js b/d2m/actions/register-user.test.js
index 5cee80b..96c73aa 100644
--- a/d2m/actions/register-user.test.js
+++ b/d2m/actions/register-user.test.js
@@ -22,6 +22,26 @@ test("member2state: without member nick or avatar", async t => {
)
})
+test("member2state: with global name, without member nick or avatar", async t => {
+ t.deepEqual(
+ await _memberToStateContent(testData.member.papiophidian.user, testData.member.papiophidian, testData.guild.general.id),
+ {
+ avatar_url: "mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX",
+ displayname: "PapiOphidian",
+ membership: "join",
+ "moe.cadence.ooye.member": {
+ avatar: "/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024"
+ },
+ "uk.half-shot.discord.member": {
+ bot: false,
+ displayColor: 1579292,
+ id: "320067006521147393",
+ username: "@papiophidian"
+ }
+ }
+ )
+})
+
test("member2state: with member nick and avatar", async t => {
t.deepEqual(
await _memberToStateContent(testData.member.sheep.user, testData.member.sheep, testData.guild.general.id),
diff --git a/test/data.js b/test/data.js
index ea9f66a..1a526aa 100644
--- a/test/data.js
+++ b/test/data.js
@@ -222,6 +222,40 @@ module.exports = {
},
mute: false,
deaf: false
+ },
+ papiophidian: {
+ avatar: null,
+ communication_disabled_until: null,
+ flags: 0,
+ joined_at: "2018-08-05T09:40:47.076000+00:00",
+ nick: null,
+ pending: false,
+ premium_since: "2021-09-30T18:58:44.996000+00:00",
+ roles: [
+ "475599410068324352",
+ "475599471049310208",
+ "497586624390234112",
+ "613685290938138625",
+ "475603310955593729",
+ "1151970058730487898",
+ "1151970058730487901"
+ ],
+ unusual_dm_activity_until: null,
+ user: {
+ id: "320067006521147393",
+ username: "papiophidian",
+ avatar: "5fc4ad85c1ea876709e9a7d3374a78a1",
+ discriminator: "0",
+ public_flags: 4194880,
+ flags: 4194880,
+ banner: "a_6f311cf6a3851a98e2fa0335af85b1d1",
+ accent_color: 1579292,
+ global_name: "PapiOphidian",
+ avatar_decoration_data: null,
+ banner_color: "#18191c"
+ },
+ mute: false,
+ deaf: false
}
},
pins: {
diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql
index f5cfb5c..f8039cc 100644
--- a/test/ooye-test-data.sql
+++ b/test/ooye-test-data.sql
@@ -75,7 +75,8 @@ INSERT INTO file (discord_url, mxc_url) VALUES
('https://cdn.discordapp.com/emojis/230201364309868544.png', 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'),
('https://cdn.discordapp.com/emojis/393635038903926784.gif', 'mxc://cadence.moe/WbYqNlACRuicynBfdnPYtmvc'),
('https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg', 'mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR'),
-('https://cdn.discordapp.com/emojis/1125827250609201255.png', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP');
+('https://cdn.discordapp.com/emojis/1125827250609201255.png', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP'),
+('https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024', 'mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX');
INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES
('230201364309868544', 'hippo', 0, 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'),
From d9d672bffd92b1de49a7e3bc5a06b21979a0e3d2 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Sat, 14 Oct 2023 02:15:21 +1300
Subject: [PATCH 023/480] d->m: Make role mentions really pretty
---
d2m/converters/message-to-event.js | 29 +++++++---
d2m/converters/message-to-event.test.js | 12 ++++
test/data.js | 76 ++++++++++++++++++++++++-
3 files changed, 109 insertions(+), 8 deletions(-)
diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js
index 7c4a998..c0c937e 100644
--- a/d2m/converters/message-to-event.js
+++ b/d2m/converters/message-to-event.js
@@ -17,7 +17,12 @@ const reg = require("../../matrix/read-registration")
const userRegex = reg.namespaces.users.map(u => new RegExp(u.regex))
-function getDiscordParseCallbacks(message, useHTML) {
+/**
+ * @param {DiscordTypes.APIMessage} message
+ * @param {DiscordTypes.APIGuild} guild
+ * @param {boolean} useHTML
+ */
+function getDiscordParseCallbacks(message, guild, useHTML) {
return {
/** @param {{id: string, type: "discordUser"}} node */
user: node => {
@@ -53,8 +58,18 @@ function getDiscordParseCallbacks(message, useHTML) {
return `:${node.name}:`
}
},
- role: node =>
- "@&" + node.id,
+ role: node => {
+ const role = guild.roles.find(r => r.id === node.id)
+ if (!role) {
+ return "@&" + node.id // fallback for if the cache breaks. if this happens, fix discord-packets.js to store the role info.
+ } else if (useHTML && role.color) {
+ return `@${role.name}`
+ } else if (useHTML) {
+ return `@${role.name}`
+ } else {
+ return `@${role.name}:`
+ }
+ },
everyone: node =>
"@room",
here: node =>
@@ -160,11 +175,11 @@ async function messageToEvent(message, guild, options = {}, di) {
}))
let html = markdown.toHTML(content, {
- discordCallback: getDiscordParseCallbacks(message, true)
+ discordCallback: getDiscordParseCallbacks(message, guild, true)
}, null, null)
let body = markdown.toHTML(content, {
- discordCallback: getDiscordParseCallbacks(message, false),
+ discordCallback: getDiscordParseCallbacks(message, guild, false),
discordOnly: true,
escapeHTML: false,
}, null, null)
@@ -223,10 +238,10 @@ async function messageToEvent(message, guild, options = {}, di) {
if (repliedToContent == "") repliedToContent = "[Media]"
else if (!repliedToContent) repliedToContent = "[Replied-to message content wasn't provided by Discord]"
const repliedToHtml = markdown.toHTML(repliedToContent, {
- discordCallback: getDiscordParseCallbacks(message, true)
+ discordCallback: getDiscordParseCallbacks(message, guild, true)
}, null, null)
const repliedToBody = markdown.toHTML(repliedToContent, {
- discordCallback: getDiscordParseCallbacks(message, false),
+ discordCallback: getDiscordParseCallbacks(message, guild, false),
discordOnly: true,
escapeHTML: false,
}, null, null)
diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js
index 4b4b6a0..8e2ba53 100644
--- a/d2m/converters/message-to-event.test.js
+++ b/d2m/converters/message-to-event.test.js
@@ -73,6 +73,18 @@ test("message2event: simple room mention", async t => {
}])
})
+test("message2event: simple role mentions", async t => {
+ const events = await messageToEvent(data.message.simple_role_mentions, data.guild.general, {})
+ t.deepEqual(events, [{
+ $type: "m.room.message",
+ "m.mentions": {},
+ msgtype: "m.text",
+ body: "I'm just @!!DLCS!!: testing a few role pings @Master Wonder Mage: don't mind me",
+ format: "org.matrix.custom.html",
+ formatted_body: `I'm just @!!DLCS!! testing a few role pings @Master Wonder Mage don't mind me`
+ }])
+})
+
test("message2event: simple message link", async t => {
const events = await messageToEvent(data.message.simple_message_link, data.guild.general, {})
t.deepEqual(events, [{
diff --git a/test/data.js b/test/data.js
index 1a526aa..3d3f1f0 100644
--- a/test/data.js
+++ b/test/data.js
@@ -119,7 +119,51 @@ module.exports = {
}
],
premium_subscription_count: 14,
- roles: [],
+ roles: [
+ {
+ version: 1696964862461,
+ unicode_emoji: null,
+ tags: {},
+ position: 22,
+ permissions: '0',
+ name: 'Master Wonder Mage',
+ mentionable: true,
+ managed: false,
+ id: '503685967463448616',
+ icon: null,
+ hoist: false,
+ flags: 0,
+ color: 0
+ }, {
+ version: 1696964862776,
+ unicode_emoji: null,
+ tags: {},
+ position: 131,
+ permissions: '0',
+ name: '!!DLCS!!',
+ mentionable: true,
+ managed: false,
+ id: '212762309364285440',
+ icon: null,
+ hoist: true,
+ flags: 0,
+ color: 11076095
+ }, {
+ version: 1696964862698,
+ unicode_emoji: '🍂',
+ tags: {},
+ position: 102,
+ permissions: '0',
+ name: 'corporate overlord',
+ mentionable: false,
+ managed: false,
+ id: '217013981053845504',
+ icon: null,
+ hoist: true,
+ flags: 0,
+ color: 16745267
+ }
+ ],
discovery_splash: null,
default_message_notifications: 1,
region: "deprecated",
@@ -434,6 +478,36 @@ module.exports = {
attachments: [],
guild_id: "112760669178241024"
},
+ simple_role_mentions: {
+ id: "1162374402785153106",
+ type: 0,
+ content: "I'm just <@&212762309364285440> testing a few role pings <@&503685967463448616> don't mind me",
+ channel_id: "160197704226439168",
+ author: {
+ id: "772659086046658620",
+ username: "cadence.worm",
+ avatar: "4b5c4b28051144e4c111f0113a0f1cf1",
+ discriminator: "0",
+ public_flags: 0,
+ flags: 0,
+ banner: null,
+ accent_color: null,
+ global_name: "cadence",
+ avatar_decoration_data: null,
+ banner_color: null
+ },
+ attachments: [],
+ embeds: [],
+ mentions: [],
+ mention_roles: [ "212762309364285440", "503685967463448616" ],
+ pinned: false,
+ mention_everyone: false,
+ tts: false,
+ timestamp: "2023-10-13T13:00:53.496000+00:00",
+ edited_timestamp: null,
+ flags: 0,
+ components: []
+ },
simple_message_link: {
id: "1126788210308161626",
type: 0,
From 1016fb1d6703f1eff6a8d9e5fd60145d98b5115f Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Sat, 14 Oct 2023 17:23:55 +1300
Subject: [PATCH 024/480] Always use OOYE bot to send thread start context
---
d2m/actions/send-message.js | 2 +-
d2m/converters/message-to-event.js | 3 ++-
d2m/converters/message-to-event.test.js | 1 +
m2d/converters/emoji.js | 1 +
4 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/d2m/actions/send-message.js b/d2m/actions/send-message.js
index a0027b0..082cce4 100644
--- a/d2m/actions/send-message.js
+++ b/d2m/actions/send-message.js
@@ -40,7 +40,7 @@ async function sendMessage(message, guild) {
}
for (const event of events) {
const eventType = event.$type
- if (event.$sender) senderMxid = event.$sender
+ if ("$sender" in event) senderMxid = event.$sender
/** @type {Pick> & { $type?: string, $sender?: string }} */
const eventWithoutType = {...event}
delete eventWithoutType.$type
diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js
index c0c937e..1a063a5 100644
--- a/d2m/converters/message-to-event.js
+++ b/d2m/converters/message-to-event.js
@@ -108,7 +108,8 @@ async function messageToEvent(message, guild, options = {}, di) {
const event = await di.api.getEvent(roomID, eventID)
return [{
...event.content,
- $type: event.type
+ $type: event.type,
+ $sender: null
}]
}
diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js
index 8e2ba53..8f89a43 100644
--- a/d2m/converters/message-to-event.test.js
+++ b/d2m/converters/message-to-event.test.js
@@ -485,6 +485,7 @@ test("message2event: thread start message reference", async t => {
})
t.deepEqual(events, [{
$type: "m.room.message",
+ $sender: null,
msgtype: "m.text",
body: "layer 4",
"m.mentions": {}
diff --git a/m2d/converters/emoji.js b/m2d/converters/emoji.js
index 214022f..2c39a86 100644
--- a/m2d/converters/emoji.js
+++ b/m2d/converters/emoji.js
@@ -36,6 +36,7 @@ function encodeEmoji(input, shortcode) {
const forceTrimmedList = [
"%F0%9F%91%8D", // 👍
+ "%F0%9F%91%8E", // 👎️
"%E2%AD%90", // ⭐
"%F0%9F%90%88", // 🐈
"%E2%9D%93", // ❓
From b7f90db20afd78d324216fe79033bccfbc431438 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Sat, 14 Oct 2023 19:27:45 +1300
Subject: [PATCH 025/480] Fix reply preview "undefined" on embed description
---
d2m/converters/user-to-mxid.test.js | 2 +-
m2d/converters/event-to-message.js | 2 +-
m2d/converters/event-to-message.test.js | 80 +++++++++++++++++++++++++
test/ooye-test-data.sql | 13 ++--
4 files changed, 91 insertions(+), 6 deletions(-)
diff --git a/d2m/converters/user-to-mxid.test.js b/d2m/converters/user-to-mxid.test.js
index 1b31260..e709473 100644
--- a/d2m/converters/user-to-mxid.test.js
+++ b/d2m/converters/user-to-mxid.test.js
@@ -17,7 +17,7 @@ test("user2name: works on emojis", t => {
})
test("user2name: works on single emoji at the end", t => {
- t.equal(userToSimName({username: "Amanda 🎵", discriminator: "2192"}), "amanda")
+ t.equal(userToSimName({username: "Melody 🎵", discriminator: "2192"}), "melody")
})
test("user2name: works on crazy name", t => {
diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js
index e7bbda5..5f6f3e6 100644
--- a/m2d/converters/event-to-message.js
+++ b/m2d/converters/event-to-message.js
@@ -352,7 +352,7 @@ async function eventToMessage(event, guild, di) {
const contentPreviewChunks = chunk(
entities.decodeHTML5Strict( // Remove entities like & "
repliedToContent.replace(/.*<\/mx-reply>/, "") // Remove everything before replies, so just use the actual message body
- .replace(/.*?<\/blockquote>/, "") // If the message starts with a blockquote, don't count it and use the message body afterwards
+ .replace(/^\s*.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards
.replace(/(?:\n|
)+/g, " ") // Should all be on one line
.replace(/]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.)
.replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting.
diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js
index 1a1c3f0..074da1d 100644
--- a/m2d/converters/event-to-message.test.js
+++ b/m2d/converters/event-to-message.test.js
@@ -813,6 +813,86 @@ test("event2message: should include a reply preview when message ends with a blo
)
})
+test("event2message: should include a reply preview when replying to a description-only bot embed", async t => {
+ t.deepEqual(
+ await eventToMessage({
+ type: "m.room.message",
+ sender: "@cadence:cadence.moe",
+ content: {
+ msgtype: "m.text",
+ body: "> <@_ooye_amanda:cadence.moe> > It looks like this queue has ended.\n\nso you're saying on matrix side I would have to edit ^this^ to add \"Timed out\" before the blockquote?",
+ format: "org.matrix.custom.html",
+ formatted_body: "In reply to @_ooye_amanda:cadence.moe
It looks like this queue has ended.
so you're saying on matrix side I would have to edit ^this^ to add "Timed out" before the blockquote?",
+ "m.relates_to": {
+ "m.in_reply_to": {
+ event_id: "$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ"
+ }
+ }
+ },
+ event_id: "$qCOlszCawu5hlnF2a2PGyXeGGvtoNJdXyRAEaTF0waA",
+ room_id: "!CzvdIdUQXgUjDVKxeU:cadence.moe"
+ }, data.guild.general, {
+ api: {
+ getEvent: mockGetEvent(t, "!CzvdIdUQXgUjDVKxeU:cadence.moe", "$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ", {
+ type: "m.room.message",
+ room_id: "!edUxjVdzgUvXDUIQCK:cadence.moe",
+ sender: "@_ooye_amanda:cadence.moe",
+ content: {
+ "m.mentions": {},
+ msgtype: "m.notice",
+ body: "> Now Playing: [**LOADING**](https://amanda.moe)\n" +
+ "> \n" +
+ "> `[====[LOADING]=====]`",
+ format: "org.matrix.custom.html",
+ formatted_body: 'Now Playing: LOADING
[====[LOADING]=====]
'
+ },
+ unsigned: {
+ "m.relations": {
+ "m.replace": {
+ type: "m.room.message",
+ room_id: "!edUxjVdzgUvXDUIQCK:cadence.moe",
+ sender: "@_ooye_amanda:cadence.moe",
+ content: {
+ "m.mentions": {},
+ msgtype: "m.notice",
+ body: "* > It looks like this queue has ended.",
+ format: "org.matrix.custom.html",
+ formatted_body: "* It looks like this queue has ended.
",
+ "m.new_content": {
+ "m.mentions": {},
+ msgtype: "m.notice",
+ body: "> It looks like this queue has ended.",
+ format: "org.matrix.custom.html",
+ formatted_body: "It looks like this queue has ended.
"
+ },
+ "m.relates_to": {
+ rel_type: "m.replace",
+ event_id: "$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ"
+ }
+ },
+ event_id: "$nrLF310vALFIXPNk6MEIy0lYiGXi210Ok0DATSaF5jQ",
+ user_id: "@_ooye_amanda:cadence.moe",
+ }
+ },
+ user_id: "@_ooye_amanda:cadence.moe",
+ }
+ })
+ }
+ }),
+ {
+ messagesToDelete: [],
+ messagesToEdit: [],
+ messagesToSend: [{
+ username: "cadence [they]",
+ content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/497161350934560778/1162625810109317170 <@1109360903096369153>:"
+ + "\n> It looks like this queue has ended."
+ + `\nso you're saying on matrix side I would have to edit ^this^ to add "Timed out" before the blockquote?`,
+ avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU"
+ }]
+ }
+ )
+})
+
test("event2message: entities are not escaped in main message or reply preview", async t => {
// Intended result: Testing? in italics, followed by the sequence "':.`[]&things
t.deepEqual(
diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql
index f8039cc..4fba480 100644
--- a/test/ooye-test-data.sql
+++ b/test/ooye-test-data.sql
@@ -18,7 +18,8 @@ INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES
('771520384671416320', 'bojack_horseman', '_ooye_bojack_horseman', '@_ooye_bojack_horseman:cadence.moe'),
('112890272819507200', '.wing.', '_ooye_.wing.', '@_ooye_.wing.:cadence.moe'),
('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'),
-('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe');;
+('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe'),
+('1109360903096369153', 'amanda', '_ooye_amanda', '@_ooye_amanda:cadence.moe');
INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES
('@_ooye_bojack_horseman:cadence.moe', '!hYnGGlPHlbujVVfktC:cadence.moe', NULL);
@@ -38,7 +39,8 @@ INSERT INTO message_channel (message_id, channel_id) VALUES
('1145688633186193479', '1100319550446252084'),
('1145688633186193480', '1100319550446252084'),
('1145688633186193481', '1100319550446252084'),
-('1162005526675193909', '1162005314908999790');
+('1162005526675193909', '1162005314908999790'),
+('1162625810109317170', '497161350934560778');
INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES
('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 1),
@@ -60,7 +62,9 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part
('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs', 'm.room.message', 'm.text', '1145688633186193479', 0, 0),
('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193480', 0, 0),
('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193481', 1, 0),
-('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0);
+('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0),
+('$0wEdIP8fhTq-P68xwo_gyUw-Zv0KA2aS2tfhdFSrLZc', 'm.room.message', 'm.text', '1162625810109317170', 1, 1),
+('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 1);
INSERT INTO file (discord_url, mxc_url) VALUES
('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'),
@@ -91,7 +95,8 @@ INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES
('!BpMdOUkWWhFxmTrENV:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'malformed mxc'),
('!fGgIymcYWOqjbSRUdV:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'),
('!BnKuBPCvyfOkhcUjEu:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'),
-('!maggESguZBqGBZtSnr:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU');
+('!maggESguZBqGBZtSnr:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'),
+('!CzvdIdUQXgUjDVKxeU:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU');
INSERT INTO "auto_emoji" ("name","emoji_id","guild_id") VALUES
('L1','1144820033948762203','529176156398682115'),
From c24752625d28947ee35a769557f020141480bd98 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Sat, 14 Oct 2023 22:08:10 +1300
Subject: [PATCH 026/480] Split part and reaction_part
Now, reactions should always end up on the bottom of a message group,
instead of sometimes being in the middle.
---
d2m/actions/add-reaction.js | 2 +-
d2m/actions/edit-message.js | 24 ++++++----
d2m/actions/remove-reaction.js | 2 +-
d2m/actions/send-message.js | 10 ++--
d2m/converters/edit-to-changes.js | 30 ++++++------
d2m/converters/edit-to-changes.test.js | 25 ++++------
.../0007-split-part-and-reaction-part.sql | 24 ++++++++++
db/orm-defs.d.ts | 1 +
m2d/actions/add-reaction.js | 2 +-
m2d/actions/send-event.js | 3 +-
scripts/check-migrate.js | 15 ++++++
test/ooye-test-data.sql | 46 +++++++++----------
test/test.js | 8 ++--
13 files changed, 121 insertions(+), 71 deletions(-)
create mode 100644 db/migrations/0007-split-part-and-reaction-part.sql
create mode 100644 scripts/check-migrate.js
diff --git a/d2m/actions/add-reaction.js b/d2m/actions/add-reaction.js
index f962a73..b131f13 100644
--- a/d2m/actions/add-reaction.js
+++ b/d2m/actions/add-reaction.js
@@ -21,7 +21,7 @@ async function addReaction(data) {
const user = data.member?.user
assert.ok(user && user.username)
- const parentID = select("event_message", "event_id", {message_id: data.message_id, part: 0}).pluck().get() // 0 = primary
+ const parentID = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get()
if (!parentID) return // Nothing can be done if the parent message was never bridged.
assert.equal(typeof parentID, "string")
diff --git a/d2m/actions/edit-message.js b/d2m/actions/edit-message.js
index a1538b9..2a08526 100644
--- a/d2m/actions/edit-message.js
+++ b/d2m/actions/edit-message.js
@@ -12,7 +12,7 @@ const api = sync.require("../../matrix/api")
* @param {import("discord-api-types/v10").APIGuild} guild
*/
async function editMessage(message, guild) {
- const {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promoteEvent, promoteNextEvent} = await editToChanges.editToChanges(message, guild, api)
+ const {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api)
// 1. Replace all the things.
for (const {oldID, newContent} of eventsToReplace) {
@@ -36,24 +36,30 @@ async function editMessage(message, guild) {
}
// 3. Consistency: Ensure there is exactly one part = 0
- let eventPart = 1
- if (promoteEvent) {
- db.prepare("UPDATE event_message SET part = 0 WHERE event_id = ?").run(promoteEvent)
- } else if (promoteNextEvent) {
- eventPart = 0
+ const sendNewEventParts = new Set()
+ for (const promotion of promotions) {
+ if ("eventID" in promotion) {
+ db.prepare(`UPDATE event_message SET ${promotion.column} = 0 WHERE event_id = ?`).run(promotion.eventID)
+ } else if ("nextEvent" in promotion) {
+ sendNewEventParts.add(promotion.column)
+ }
}
// 4. Send all the things.
+ if (eventsToSend.length) {
+ db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
+ }
for (const content of eventsToSend) {
const eventType = content.$type
/** @type {Pick> & { $type?: string }} */
const contentWithoutType = {...content}
delete contentWithoutType.$type
+ delete contentWithoutType.$sender
+ const part = sendNewEventParts.has("part") && eventsToSend[0] === content ? 0 : 1
+ const reactionPart = sendNewEventParts.has("reaction_part") && eventsToSend[eventsToSend.length - 1] === content ? 0 : 1
const eventID = await api.sendEvent(roomID, eventType, contentWithoutType, senderMxid)
- db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, 1)").run(eventID, eventType, content.msgtype || null, message.id, eventPart) // part 1 = supporting; source 1 = discord
-
- eventPart = 1
+ db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, content.msgtype || null, message.id, part, reactionPart) // source 1 = discord
}
}
diff --git a/d2m/actions/remove-reaction.js b/d2m/actions/remove-reaction.js
index 3047c32..b2dba0f 100644
--- a/d2m/actions/remove-reaction.js
+++ b/d2m/actions/remove-reaction.js
@@ -20,7 +20,7 @@ const converter = sync.require("../converters/remove-reaction")
async function removeSomeReactions(data) {
const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
if (!roomID) return
- const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, part: 0}).pluck().get()
+ const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get()
if (!eventIDForMessage) return
/** @type {Ty.Pagination>} */
diff --git a/d2m/actions/send-message.js b/d2m/actions/send-message.js
index 082cce4..b59fc7f 100644
--- a/d2m/actions/send-message.js
+++ b/d2m/actions/send-message.js
@@ -33,12 +33,14 @@ async function sendMessage(message, guild) {
const events = await messageToEvent.messageToEvent(message, guild, {}, {api})
const eventIDs = []
- let eventPart = 0 // 0 is primary, 1 is supporting
if (events.length) {
db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
if (senderMxid) api.sendTyping(roomID, false, senderMxid)
}
for (const event of events) {
+ const part = event === events[0] ? 0 : 1
+ const reactionPart = event === events[events.length - 1] ? 0 : 1
+
const eventType = event.$type
if ("$sender" in event) senderMxid = event.$sender
/** @type {Pick> & { $type?: string, $sender?: string }} */
@@ -48,12 +50,14 @@ async function sendMessage(message, guild) {
const useTimestamp = message["backfill"] ? new Date(message.timestamp).getTime() : undefined
const eventID = await api.sendEvent(roomID, eventType, eventWithoutType, senderMxid, useTimestamp)
- db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, eventPart) // source 1 = discord
+ db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, part, reactionPart) // source 1 = discord
// The primary event is part = 0 and has the most important and distinct information. It is used to provide reply previews, be pinned, and possibly future uses.
// The first event is chosen to be the primary part because it is usually the message text content and is more likely to be distinct.
// For example, "Reply to 'this meme made me think of you'" is more useful than "Replied to image".
- eventPart = 1
+
+ // The last event gets reaction_part = 0. Reactions are managed there because reactions are supposed to appear at the bottom.
+
eventIDs.push(eventID)
}
diff --git a/d2m/converters/edit-to-changes.js b/d2m/converters/edit-to-changes.js
index dc42708..f86921c 100644
--- a/d2m/converters/edit-to-changes.js
+++ b/d2m/converters/edit-to-changes.js
@@ -26,7 +26,7 @@ async function editToChanges(message, guild, api) {
/** @type {string?} Null if we don't have a sender in the room, which will happen if it's a webhook's message. The bridge bot will do the edit instead. */
const senderMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() || null
- const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part"], {message_id: message.id}).all()
+ const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part", "reaction_part"], {message_id: message.id}).all()
// Figure out what we will be replacing them with
@@ -83,17 +83,21 @@ async function editToChanges(message, guild, api) {
// Anything remaining in oldEventRows is present in the old version only and should be redacted.
eventsToRedact = oldEventRows
- // If events are being deleted, we might be deleting the part = 0. But we want to have a part = 0 at all times. In this case we choose an existing event to promote.
- let promoteEvent = null, promoteNextEvent = false
- if (eventsToRedact.some(e => e.part === 0)) {
- if (eventsToReplace.length) {
- // We can choose an existing event to promote. Bigger order is better.
- const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.event_subtype === "m.text")
- eventsToReplace.sort((a, b) => order(b) - order(a))
- promoteEvent = eventsToReplace[0].old.event_id
- } else {
- // Everything is being deleted. Whatever gets sent in their place will be the new part = 0.
- promoteNextEvent = true
+ // We want to maintain exactly one part = 0 and one reaction_part = 0 database row at all times.
+ /** @type {({column: string, eventID: string} | {column: string, nextEvent: true})[]} */
+ const promotions = []
+ for (const column of ["part", "reaction_part"]) {
+ // If no events with part = 0 exist (or will exist), we need to do some management.
+ if (!eventsToReplace.some(e => e.old[column] === 0)) {
+ if (eventsToReplace.length) {
+ // We can choose an existing event to promote. Bigger order is better.
+ const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.event_subtype === "m.text")
+ eventsToReplace.sort((a, b) => order(b) - order(a))
+ promotions.push({column, eventID: eventsToReplace[0].old.event_id})
+ } else {
+ // No existing events to promote, but new events are being sent. Whatever gets sent will be the next part = 0.
+ promotions.push({column, nextEvent: true})
+ }
}
}
@@ -117,7 +121,7 @@ async function editToChanges(message, guild, api) {
eventsToRedact = eventsToRedact.map(e => e.event_id)
eventsToReplace = eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.newFallbackContent, e.newInnerContent)}))
- return {roomID, eventsToReplace, eventsToRedact, eventsToSend, senderMxid, promoteEvent, promoteNextEvent}
+ return {roomID, eventsToReplace, eventsToRedact, eventsToSend, senderMxid, promotions}
}
/**
diff --git a/d2m/converters/edit-to-changes.test.js b/d2m/converters/edit-to-changes.test.js
index 8c67f6a..7c29787 100644
--- a/d2m/converters/edit-to-changes.test.js
+++ b/d2m/converters/edit-to-changes.test.js
@@ -4,7 +4,7 @@ const data = require("../../test/data")
const Ty = require("../../types")
test("edit2changes: edit by webhook", async t => {
- const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promoteEvent, promoteNextEvent} = await editToChanges(data.message_update.edit_by_webhook, data.guild.general, {})
+ const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.edit_by_webhook, data.guild.general, {})
t.deepEqual(eventsToRedact, [])
t.deepEqual(eventsToSend, [])
t.deepEqual(eventsToReplace, [{
@@ -27,12 +27,11 @@ test("edit2changes: edit by webhook", async t => {
}
}])
t.equal(senderMxid, null)
- t.equal(promoteEvent, null)
- t.equal(promoteNextEvent, false)
+ t.deepEqual(promotions, [])
})
test("edit2changes: bot response", async t => {
- const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promoteEvent, promoteNextEvent} = await editToChanges(data.message_update.bot_response, data.guild.general, {
+ const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.bot_response, data.guild.general, {
async getJoinedMembers(roomID) {
t.equal(roomID, "!hYnGGlPHlbujVVfktC:cadence.moe")
return new Promise(resolve => {
@@ -84,21 +83,19 @@ test("edit2changes: bot response", async t => {
}
}])
t.equal(senderMxid, "@_ooye_bojack_horseman:cadence.moe")
- t.equal(promoteEvent, null)
- t.equal(promoteNextEvent, false)
+ t.deepEqual(promotions, [])
})
test("edit2changes: remove caption from image", async t => {
- const {eventsToRedact, eventsToReplace, eventsToSend, promoteEvent, promoteNextEvent} = await editToChanges(data.message_update.removed_caption_from_image, data.guild.general, {})
+ const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.removed_caption_from_image, data.guild.general, {})
t.deepEqual(eventsToRedact, ["$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA"])
t.deepEqual(eventsToSend, [])
t.deepEqual(eventsToReplace, [])
- t.equal(promoteEvent, "$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI")
- t.equal(promoteNextEvent, false)
+ t.deepEqual(promotions, [{column: "part", eventID: "$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI"}])
})
test("edit2changes: change file type", async t => {
- const {eventsToRedact, eventsToReplace, eventsToSend, promoteEvent, promoteNextEvent} = await editToChanges(data.message_update.changed_file_type, data.guild.general, {})
+ const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.changed_file_type, data.guild.general, {})
t.deepEqual(eventsToRedact, ["$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ"])
t.deepEqual(eventsToSend, [{
$type: "m.room.message",
@@ -109,12 +106,11 @@ test("edit2changes: change file type", async t => {
msgtype: "m.text"
}])
t.deepEqual(eventsToReplace, [])
- t.equal(promoteEvent, null)
- t.equal(promoteNextEvent, true)
+ t.deepEqual(promotions, [{column: "part", nextEvent: true}, {column: "reaction_part", nextEvent: true}])
})
test("edit2changes: add caption back to that image", async t => {
- const {eventsToRedact, eventsToReplace, eventsToSend, promoteEvent, promoteNextEvent} = await editToChanges(data.message_update.added_caption_to_image, data.guild.general, {})
+ const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.added_caption_to_image, data.guild.general, {})
t.deepEqual(eventsToRedact, [])
t.deepEqual(eventsToSend, [{
$type: "m.room.message",
@@ -123,8 +119,7 @@ test("edit2changes: add caption back to that image", async t => {
"m.mentions": {}
}])
t.deepEqual(eventsToReplace, [])
- t.equal(promoteEvent, null)
- t.equal(promoteNextEvent, false)
+ t.deepEqual(promotions, [])
})
test("edit2changes: stickers and attachments are not changed, only the content can be edited", async t => {
diff --git a/db/migrations/0007-split-part-and-reaction-part.sql b/db/migrations/0007-split-part-and-reaction-part.sql
new file mode 100644
index 0000000..4aad6c1
--- /dev/null
+++ b/db/migrations/0007-split-part-and-reaction-part.sql
@@ -0,0 +1,24 @@
+BEGIN TRANSACTION;
+
+-- Add column reaction_part to event_message, copying the existing value from part
+
+CREATE TABLE "new_event_message" (
+ "event_id" TEXT NOT NULL,
+ "event_type" TEXT,
+ "event_subtype" TEXT,
+ "message_id" TEXT NOT NULL,
+ "part" INTEGER NOT NULL,
+ "reaction_part" INTEGER NOT NULL,
+ "source" INTEGER NOT NULL,
+ PRIMARY KEY("message_id","event_id")
+) WITHOUT ROWID;
+
+INSERT INTO new_event_message SELECT event_id, event_type, event_subtype, message_id, part, part, source FROM event_message;
+
+DROP TABLE event_message;
+
+ALTER TABLE new_event_message RENAME TO event_message;
+
+COMMIT;
+
+VACUUM;
diff --git a/db/orm-defs.d.ts b/db/orm-defs.d.ts
index 0714e0b..ec8d498 100644
--- a/db/orm-defs.d.ts
+++ b/db/orm-defs.d.ts
@@ -14,6 +14,7 @@ export type Models = {
event_type: string | null
event_subtype: string | null
part: number
+ reaction_part: number
source: number
}
diff --git a/m2d/actions/add-reaction.js b/m2d/actions/add-reaction.js
index e6a94d9..cfd471b 100644
--- a/m2d/actions/add-reaction.js
+++ b/m2d/actions/add-reaction.js
@@ -16,7 +16,7 @@ const emoji = sync.require("../converters/emoji")
async function addReaction(event) {
const channelID = select("channel_room", "channel_id", {room_id: event.room_id}).pluck().get()
if (!channelID) return // We just assume the bridge has already been created
- const messageID = select("event_message", "message_id", {event_id: event.content["m.relates_to"].event_id, part: 0}).pluck().get() // 0 = primary
+ const messageID = select("event_message", "message_id", {event_id: event.content["m.relates_to"].event_id}, "ORDER BY reaction_part").pluck().get()
if (!messageID) return // Nothing can be done if the parent message was never bridged.
const key = event.content["m.relates_to"].key
diff --git a/m2d/actions/send-event.js b/m2d/actions/send-event.js
index c1e3ba3..13b5650 100644
--- a/m2d/actions/send-event.js
+++ b/m2d/actions/send-event.js
@@ -98,9 +98,10 @@ async function sendEvent(event) {
}
for (const message of messagesToSend) {
+ const reactionPart = messagesToEdit.length === 0 && message === messagesToSend[messagesToSend.length - 1] ? 0 : 1
const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message, threadID)
db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(messageResponse.id, threadID || channelID)
- db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content["msgtype"] || null, messageResponse.id, eventPart) // source 0 = matrix
+ db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content["msgtype"] || null, messageResponse.id, eventPart, reactionPart) // source 0 = matrix
eventPart = 1
messageResponses.push(messageResponse)
diff --git a/scripts/check-migrate.js b/scripts/check-migrate.js
new file mode 100644
index 0000000..308ea87
--- /dev/null
+++ b/scripts/check-migrate.js
@@ -0,0 +1,15 @@
+// @ts-check
+
+// Trigger the database migration flow and exit after committing.
+// You can use this to run migrations locally and check the result using sqlitebrowser.
+
+const sqlite = require("better-sqlite3")
+
+const config = require("../config")
+const passthrough = require("../passthrough")
+const db = new sqlite("db/ooye.db")
+const migrate = require("../db/migrate")
+
+Object.assign(passthrough, {config, db })
+
+migrate.migrate(db)
diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql
index 4fba480..0627d08 100644
--- a/test/ooye-test-data.sql
+++ b/test/ooye-test-data.sql
@@ -42,29 +42,29 @@ INSERT INTO message_channel (message_id, channel_id) VALUES
('1162005526675193909', '1162005314908999790'),
('1162625810109317170', '497161350934560778');
-INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES
-('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 1),
-('$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4', 'm.room.message', 'm.text', '1128118177155526666', 0, 0),
-('$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10', 'm.room.message', 'm.text', '1141619794500649020', 0, 1),
-('$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY', 'm.room.message', 'm.text', '1141206225632112650', 0, 1),
-('$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA', 'm.room.message', 'm.text', '1141501302736695316', 0, 1),
-('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', 1, 1),
-('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ', 'm.room.message', 'm.image', '1141501302736695317', 0, 1),
-('$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M', 'm.room.message', 'm.text', '1128084851279536279', 0, 1),
-('$YUJFa5j0ZJe7PUvD2DykRt9g51RoadUEYmuJLdSEbJ0', 'm.room.message', 'm.image', '1128084851279536279', 1, 1),
-('$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q', 'm.room.message', 'm.text', '1128084748338741392', 0, 1),
-('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', 0, 1),
-('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', 0, 1),
-('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 1, 0),
-('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 1, 0),
-('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', 0, 1),
-('$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8', 'm.room.message', 'm.text', '1144874214311067708', 0, 0),
-('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs', 'm.room.message', 'm.text', '1145688633186193479', 0, 0),
-('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193480', 0, 0),
-('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193481', 1, 0),
-('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0),
-('$0wEdIP8fhTq-P68xwo_gyUw-Zv0KA2aS2tfhdFSrLZc', 'm.room.message', 'm.text', '1162625810109317170', 1, 1),
-('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 1);
+INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES
+('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 0, 1),
+('$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4', 'm.room.message', 'm.text', '1128118177155526666', 0, 0, 0),
+('$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10', 'm.room.message', 'm.text', '1141619794500649020', 0, 0, 1),
+('$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY', 'm.room.message', 'm.text', '1141206225632112650', 0, 0, 1),
+('$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA', 'm.room.message', 'm.text', '1141501302736695316', 0, 1, 1),
+('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', 1, 0, 1),
+('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ', 'm.room.message', 'm.image', '1141501302736695317', 0, 0, 1),
+('$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M', 'm.room.message', 'm.text', '1128084851279536279', 0, 1, 1),
+('$YUJFa5j0ZJe7PUvD2DykRt9g51RoadUEYmuJLdSEbJ0', 'm.room.message', 'm.image', '1128084851279536279', 1, 0, 1),
+('$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q', 'm.room.message', 'm.text', '1128084748338741392', 0, 0, 1),
+('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', 0, 0, 1),
+('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', 0, 1, 1),
+('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 1, 1, 0),
+('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 1, 0, 0),
+('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', 0, 0, 1),
+('$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8', 'm.room.message', 'm.text', '1144874214311067708', 0, 0, 0),
+('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs', 'm.room.message', 'm.text', '1145688633186193479', 0, 0, 0),
+('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193480', 0, 0, 0),
+('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193481', 1, 0, 0),
+('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0, 0),
+('$0wEdIP8fhTq-P68xwo_gyUw-Zv0KA2aS2tfhdFSrLZc', 'm.room.message', 'm.text', '1162625810109317170', 1, 1, 1),
+('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 0, 1);
INSERT INTO file (discord_url, mxc_url) VALUES
('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'),
diff --git a/test/test.js b/test/test.js
index 5e15a14..90e3f5a 100644
--- a/test/test.js
+++ b/test/test.js
@@ -50,16 +50,16 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
require("../matrix/file.test")
require("../matrix/read-registration.test")
require("../matrix/txnid.test")
+ require("../d2m/actions/create-room.test")
+ require("../d2m/actions/register-user.test")
+ require("../d2m/converters/edit-to-changes.test")
+ require("../d2m/converters/emoji-to-key.test")
require("../d2m/converters/message-to-event.test")
require("../d2m/converters/message-to-event.embeds.test")
- require("../d2m/converters/edit-to-changes.test")
require("../d2m/converters/pins-to-list.test")
require("../d2m/converters/remove-reaction.test")
require("../d2m/converters/thread-to-announcement.test")
require("../d2m/converters/user-to-mxid.test")
- require("../d2m/converters/emoji-to-key.test")
- require("../d2m/actions/create-room.test")
- require("../d2m/actions/register-user.test")
require("../m2d/converters/event-to-message.test")
require("../m2d/converters/utils.test")
})()
From 040e987d032bd2b28b1d84662ee8d9c5ac61bc3b Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Sat, 14 Oct 2023 23:34:02 +1300
Subject: [PATCH 027/480] Paginate removing all reactions from Matrix-side
---
d2m/actions/remove-reaction.js | 36 ++++++++++++--------
d2m/converters/remove-reaction.js | 14 ++++----
d2m/converters/remove-reaction.test.js | 46 ++++++++++++--------------
matrix/api.js | 6 +++-
4 files changed, 56 insertions(+), 46 deletions(-)
diff --git a/d2m/actions/remove-reaction.js b/d2m/actions/remove-reaction.js
index b2dba0f..95fc0aa 100644
--- a/d2m/actions/remove-reaction.js
+++ b/d2m/actions/remove-reaction.js
@@ -23,14 +23,22 @@ async function removeSomeReactions(data) {
const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get()
if (!eventIDForMessage) return
- /** @type {Ty.Pagination>} */
- const relations = await api.getRelations(roomID, eventIDForMessage, "m.annotation")
+ /** @type {Ty.Event.Outer[]} */
+ let reactions = []
+ /** @type {string | undefined} */
+ let nextBatch = undefined
+ do {
+ /** @type {Ty.Pagination>} */
+ const res = await api.getRelations(roomID, eventIDForMessage, {from: nextBatch}, "m.annotation")
+ reactions = reactions.concat(res.chunk)
+ nextBatch = res.next_batch
+ } while (nextBatch)
// Run the proper strategy and any strategy-specific database changes
const removals = await
- ( "user_id" in data ? removeReaction(data, relations)
- : "emoji" in data ? removeEmojiReaction(data, relations)
- : removeAllReactions(data, relations))
+ ( "user_id" in data ? removeReaction(data, reactions)
+ : "emoji" in data ? removeEmojiReaction(data, reactions)
+ : removeAllReactions(data, reactions))
// Redact the events and delete individual stored events in the database
for (const removal of removals) {
@@ -41,33 +49,33 @@ async function removeSomeReactions(data) {
/**
* @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData} data
- * @param {Ty.Pagination>} relations
+ * @param {Ty.Event.Outer[]} reactions
*/
-async function removeReaction(data, relations) {
+async function removeReaction(data, reactions) {
const key = await emojiToKey.emojiToKey(data.emoji)
- return converter.removeReaction(data, relations, key)
+ return converter.removeReaction(data, reactions, key)
}
/**
* @param {DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData} data
- * @param {Ty.Pagination>} relations
+ * @param {Ty.Event.Outer[]} reactions
*/
-async function removeEmojiReaction(data, relations) {
+async function removeEmojiReaction(data, reactions) {
const key = await emojiToKey.emojiToKey(data.emoji)
const discordPreferredEncoding = emoji.encodeEmoji(key, undefined)
db.prepare("DELETE FROM reaction WHERE message_id = ? AND encoded_emoji = ?").run(data.message_id, discordPreferredEncoding)
- return converter.removeEmojiReaction(data, relations, key)
+ return converter.removeEmojiReaction(data, reactions, key)
}
/**
* @param {DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data
- * @param {Ty.Pagination>} relations
+ * @param {Ty.Event.Outer[]} reactions
*/
-async function removeAllReactions(data, relations) {
+async function removeAllReactions(data, reactions) {
db.prepare("DELETE FROM reaction WHERE message_id = ?").run(data.message_id)
- return converter.removeAllReactions(data, relations)
+ return converter.removeAllReactions(data, reactions)
}
module.exports.removeSomeReactions = removeSomeReactions
diff --git a/d2m/converters/remove-reaction.js b/d2m/converters/remove-reaction.js
index 4fed269..a6c8ace 100644
--- a/d2m/converters/remove-reaction.js
+++ b/d2m/converters/remove-reaction.js
@@ -17,15 +17,15 @@ const utils = sync.require("../../m2d/converters/utils")
/**
* @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData} data
- * @param {Ty.Pagination>} relations
+ * @param {Ty.Event.Outer[]} reactions
* @param {string} key
*/
-function removeReaction(data, relations, key) {
+function removeReaction(data, reactions, key) {
/** @type {ReactionRemoveRequest[]} */
const removals = []
const wantToRemoveMatrixReaction = data.user_id === discord.application.id
- for (const event of relations.chunk) {
+ for (const event of reactions) {
const eventID = event.event_id
if (event.content["m.relates_to"].key === key) {
const lookingAtMatrixReaction = !utils.eventSenderIsFromDiscord(event.sender)
@@ -52,14 +52,14 @@ function removeReaction(data, relations, key) {
/**
* @param {DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData} data
- * @param {Ty.Pagination>} relations
+ * @param {Ty.Event.Outer[]} relations
* @param {string} key
*/
function removeEmojiReaction(data, relations, key) {
/** @type {ReactionRemoveRequest[]} */
const removals = []
- for (const event of relations.chunk) {
+ for (const event of relations) {
const eventID = event.event_id
if (event.content["m.relates_to"].key === key) {
const mxid = utils.eventSenderIsFromDiscord(event.sender) ? event.sender : null
@@ -72,11 +72,11 @@ function removeEmojiReaction(data, relations, key) {
/**
* @param {DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data
- * @param {Ty.Pagination>} relations
+ * @param {Ty.Event.Outer[]} relations
* @returns {ReactionRemoveRequest[]}
*/
function removeAllReactions(data, relations) {
- return relations.chunk.map(event => {
+ return relations.map(event => {
const eventID = event.event_id
const mxid = utils.eventSenderIsFromDiscord(event.sender) ? event.sender : null
return {eventID, mxid}
diff --git a/d2m/converters/remove-reaction.test.js b/d2m/converters/remove-reaction.test.js
index a63721f..dc6eda5 100644
--- a/d2m/converters/remove-reaction.test.js
+++ b/d2m/converters/remove-reaction.test.js
@@ -29,30 +29,28 @@ function fakeAllReactionRemoval() {
}
}
-function fakeChunk(chunk) {
- return {
- chunk: chunk.map(({sender, key}, i) => ({
- content: {
- "m.relates_to": {
- rel_type: "m.annotation",
- event_id: "$message",
- key
- }
- },
- event_id: `$reaction_${i}`,
- sender,
- type: "m.reaction",
- origin_server_ts: 0,
- room_id: "!THE_ROOM",
- unsigned: null
- }))
- }
+function fakeReactions(reactions) {
+ return reactions.map(({sender, key}, i) => ({
+ content: {
+ "m.relates_to": {
+ rel_type: "m.annotation",
+ event_id: "$message",
+ key
+ }
+ },
+ event_id: `$reaction_${i}`,
+ sender,
+ type: "m.reaction",
+ origin_server_ts: 0,
+ room_id: "!THE_ROOM",
+ unsigned: null
+ }))
}
test("remove reaction: a specific discord user's reaction is removed", t => {
const removals = removeReaction.removeReaction(
fakeSpecificReactionRemoval("820865262526005258", "🐈", null),
- fakeChunk([{key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"}]),
+ fakeReactions([{key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"}]),
"🐈"
)
t.deepEqual(removals, [{
@@ -64,7 +62,7 @@ test("remove reaction: a specific discord user's reaction is removed", t => {
test("remove reaction: a specific matrix user's reaction is removed", t => {
const removals = removeReaction.removeReaction(
fakeSpecificReactionRemoval(BRIDGE_ID, "🐈", null),
- fakeChunk([{key: "🐈", sender: "@cadence:cadence.moe"}]),
+ fakeReactions([{key: "🐈", sender: "@cadence:cadence.moe"}]),
"🐈"
)
t.deepEqual(removals, [{
@@ -77,7 +75,7 @@ test("remove reaction: a specific matrix user's reaction is removed", t => {
test("remove reaction: a specific discord user's reaction is removed when there are multiple reactions", t => {
const removals = removeReaction.removeReaction(
fakeSpecificReactionRemoval("820865262526005258", "🐈", null),
- fakeChunk([
+ fakeReactions([
{key: "🐈⬛", sender: "@_ooye_crunch_god:cadence.moe"},
{key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"},
{key: "🐈", sender: "@_ooye_extremity:cadence.moe"},
@@ -95,7 +93,7 @@ test("remove reaction: a specific discord user's reaction is removed when there
test("remove reaction: a specific reaction leads to all matrix users' reaction of the emoji being removed", t => {
const removals = removeReaction.removeReaction(
fakeSpecificReactionRemoval(BRIDGE_ID, "🐈", null),
- fakeChunk([
+ fakeReactions([
{key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"},
{key: "🐈", sender: "@cadence:cadence.moe"},
{key: "🐈⬛", sender: "@zoe:cadence.moe"},
@@ -118,7 +116,7 @@ test("remove reaction: a specific reaction leads to all matrix users' reaction o
test("remove reaction: an emoji removes all instances of the emoij from both sides", t => {
const removals = removeReaction.removeEmojiReaction(
fakeEmojiReactionRemoval("🐈", null),
- fakeChunk([
+ fakeReactions([
{key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"},
{key: "🐈", sender: "@cadence:cadence.moe"},
{key: "🐈⬛", sender: "@zoe:cadence.moe"},
@@ -145,7 +143,7 @@ test("remove reaction: an emoji removes all instances of the emoij from both sid
test("remove reaction: remove all removes all from both sides", t => {
const removals = removeReaction.removeAllReactions(
fakeAllReactionRemoval(),
- fakeChunk([
+ fakeReactions([
{key: "🐈", sender: "@_ooye_crunch_god:cadence.moe"},
{key: "🐈", sender: "@cadence:cadence.moe"},
{key: "🐈⬛", sender: "@zoe:cadence.moe"},
diff --git a/matrix/api.js b/matrix/api.js
index 7ec044a..5509930 100644
--- a/matrix/api.js
+++ b/matrix/api.js
@@ -112,12 +112,16 @@ function getJoinedMembers(roomID) {
/**
* @param {string} roomID
* @param {string} eventID
+ * @param {{from?: string, limit?: any}} pagination
* @param {string?} [relType]
* @returns {Promise>>}
*/
-function getRelations(roomID, eventID, relType) {
+function getRelations(roomID, eventID, pagination, relType) {
let path = `/client/v1/rooms/${roomID}/relations/${eventID}`
if (relType) path += `/${relType}`
+ if (!pagination.from) delete pagination.from
+ if (!pagination.limit) pagination.limit = 50 // get a little more consistency between homeservers
+ path += `?${new URLSearchParams(pagination)}`
return mreq.mreq("GET", path)
}
From fff8f0d94c344c13a84463ced6e7c748d37f0141 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Sat, 14 Oct 2023 23:58:22 +1300
Subject: [PATCH 028/480] Review and upgrade dependencies
---
package-lock.json | 162 ++++++++++++++++++++--------------------------
package.json | 8 +--
readme.md | 2 +-
3 files changed, 75 insertions(+), 97 deletions(-)
diff --git a/package-lock.json b/package-lock.json
index 3847ee1..a005645 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,9 +10,9 @@
"license": "AGPL-3.0-or-later",
"dependencies": {
"@chriscdn/promise-semaphore": "^2.0.1",
- "better-sqlite3": "^8.3.0",
+ "better-sqlite3": "^9.0.0",
"chunk-text": "^2.0.1",
- "cloudstorm": "^0.8.0",
+ "cloudstorm": "^0.9.0",
"discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3",
"entities": "^4.5.0",
"giframe": "github:cloudrac3r/giframe#v0.4.1",
@@ -25,7 +25,7 @@
"pngjs": "^7.0.0",
"prettier-bytes": "^1.0.4",
"sharp": "^0.32.6",
- "snowtransfer": "^0.8.0",
+ "snowtransfer": "^0.9.0",
"stream-mime-type": "^1.0.2",
"try-to-catch": "^3.0.1",
"turndown": "^7.1.2",
@@ -36,7 +36,7 @@
"@types/node-fetch": "^2.6.3",
"c8": "^8.0.1",
"cross-env": "^7.0.3",
- "discord-api-types": "^0.37.53",
+ "discord-api-types": "^0.37.60",
"supertape": "^8.3.0",
"tap-dot": "github:cloudrac3r/tap-dot#9dd7750ececeae3a96afba91905be812b6b2cc2d"
}
@@ -48,9 +48,9 @@
"dev": true
},
"node_modules/@chriscdn/promise-semaphore": {
- "version": "2.0.1",
- "resolved": "https://registry.npmjs.org/@chriscdn/promise-semaphore/-/promise-semaphore-2.0.1.tgz",
- "integrity": "sha512-C0Ku5DNZFbafbSRXagidIaRgzhgGmSHk4aAgPpmmHEostazBiSaMryovC/Aix3vRLNuaeGDKN/DHoNECmMD6jg=="
+ "version": "2.0.7",
+ "resolved": "https://registry.npmjs.org/@chriscdn/promise-semaphore/-/promise-semaphore-2.0.7.tgz",
+ "integrity": "sha512-xsa5SAYSBnYjqvGnzmaLca4X/RFeOl+ziCsIHl5iHkFBgE4NgWupB4z3A1rVMBM2I8TEKaah+5iu9Cm7gQu9JQ=="
},
"node_modules/@cloudcmd/stub": {
"version": "4.0.1",
@@ -152,6 +152,14 @@
"node": ">=8"
}
},
+ "node_modules/@fastify/busboy": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.0.0.tgz",
+ "integrity": "sha512-JUFJad5lv7jxj926GPgymrWQxxjPYuJNiNjNMzqT+HiuP6Vl3dk5xzG+8sTX96np0ZAluvaMzPsjhHZ5rNuNQQ==",
+ "engines": {
+ "node": ">=14"
+ }
+ },
"node_modules/@istanbuljs/schema": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz",
@@ -231,15 +239,15 @@
"dev": true
},
"node_modules/@supertape/engine-loader": {
- "version": "1.1.3",
- "resolved": "https://registry.npmjs.org/@supertape/engine-loader/-/engine-loader-1.1.3.tgz",
- "integrity": "sha512-5ilgEng0WBvMQjNJWQ/bnAA6HKgbLKxTya2C0RxFH0LYSN5faBVtgxjLDvTQ+5L+ZxjK/7ooQDDaRS1Mo0ga5Q==",
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/@supertape/engine-loader/-/engine-loader-2.0.0.tgz",
+ "integrity": "sha512-1G2MmfZnSxx546omLPAVNgvG/iqOQZGiXHnjJ2JXKvuf2lpPdDRnNm5eLl81lvEG473zE9neX979TzeFcr3Dxw==",
"dev": true,
"dependencies": {
"try-catch": "^3.0.0"
},
"engines": {
- "node": ">=14"
+ "node": ">=16"
}
},
"node_modules/@supertape/formatter-fail": {
@@ -268,9 +276,9 @@
}
},
"node_modules/@supertape/formatter-progress-bar": {
- "version": "3.0.0",
- "resolved": "https://registry.npmjs.org/@supertape/formatter-progress-bar/-/formatter-progress-bar-3.0.0.tgz",
- "integrity": "sha512-rVFAQ21eApq3TQV8taFLNcCxcGZvvOPxQC63swdmHFCp+07Dt3tvC/aFxF35NLobc3rySasGSEuPucpyoPrjfg==",
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@supertape/formatter-progress-bar/-/formatter-progress-bar-4.1.0.tgz",
+ "integrity": "sha512-MYwso7kbiTE0DaZgbiSlNOikmEcFdL4RQUu1JvnW+cS6ZLl3fqNnmvKa1a14VChKyHzfaTKYLuqToN8zgUjP2g==",
"dev": true,
"dependencies": {
"chalk": "^4.1.0",
@@ -280,7 +288,7 @@
"once": "^1.4.0"
},
"engines": {
- "node": ">=14"
+ "node": ">=16"
}
},
"node_modules/@supertape/formatter-short": {
@@ -325,19 +333,19 @@
"dev": true
},
"node_modules/@types/node": {
- "version": "18.16.5",
- "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.5.tgz",
- "integrity": "sha512-seOA34WMo9KB+UA78qaJoCO20RJzZGVXQ5Sh6FWu0g/hfT44nKXnej3/tCQl7FL97idFpBhisLYCTB50S0EirA==",
+ "version": "18.18.5",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.5.tgz",
+ "integrity": "sha512-4slmbtwV59ZxitY4ixUZdy1uRLf9eSIvBWPQxNjhHYWEtn0FryfKpyS2cvADYXTayWdKEIsJengncrVvkI4I6A==",
"dev": true
},
"node_modules/@types/node-fetch": {
- "version": "2.6.3",
- "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.3.tgz",
- "integrity": "sha512-ETTL1mOEdq/sxUtgtOhKjyB2Irra4cjxksvcMUR5Zr4n+PxVhsCD9WS46oPbHL3et9Zde7CNRr+WUNlcHvsX+w==",
+ "version": "2.6.6",
+ "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.6.tgz",
+ "integrity": "sha512-95X8guJYhfqiuVVhRFxVQcf4hW/2bCuoPwDasMf/531STFoNoWTT7YDnWdXHEZKqAGUigmpG31r2FE70LwnzJw==",
"dev": true,
"dependencies": {
"@types/node": "*",
- "form-data": "^3.0.0"
+ "form-data": "^4.0.0"
}
},
"node_modules/@types/prop-types": {
@@ -446,7 +454,8 @@
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
- "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+ "dev": true
},
"node_modules/available-typed-arrays": {
"version": "1.0.5",
@@ -512,13 +521,13 @@
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/better-sqlite3": {
- "version": "8.4.0",
- "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-8.4.0.tgz",
- "integrity": "sha512-NmsNW1CQvqMszu/CFAJ3pLct6NEFlNfuGM6vw72KHkjOD1UDnL96XNN1BMQc1hiHo8vE2GbOWQYIpZ+YM5wrZw==",
+ "version": "9.0.0",
+ "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-9.0.0.tgz",
+ "integrity": "sha512-lDxQ9qg/XuUHZG6xzrQaMHkNWl37t35/LPB/VJGV8DdScSuGFNfFSqgscXEd8UIuyk/d9wU8iaMxQa4If5Wqog==",
"hasInstallScript": true,
"dependencies": {
"bindings": "^1.5.0",
- "prebuild-install": "^7.1.0"
+ "prebuild-install": "^7.1.1"
}
},
"node_modules/bindings": {
@@ -631,17 +640,6 @@
"ieee754": "^1.2.1"
}
},
- "node_modules/busboy": {
- "version": "1.6.0",
- "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
- "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
- "dependencies": {
- "streamsearch": "^1.1.0"
- },
- "engines": {
- "node": ">=10.16.0"
- }
- },
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -784,14 +782,14 @@
}
},
"node_modules/cloudstorm": {
- "version": "0.8.3",
- "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.8.3.tgz",
- "integrity": "sha512-4c2rqFFvzM4P3pcnjnGUlYuyBjx/xnMew6imB0sFwmNLITLCTLYa3qGkrnhI1g/tM0fqg+Gr+EmDHiDZfEr9LQ==",
+ "version": "0.9.0",
+ "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.9.0.tgz",
+ "integrity": "sha512-n5M5TVnvm/X5vdNKy85q8muMregnvPWxv7HGSDCChL/FReOh2PGOm0FZJVm4hcB+KIM07KmiJTiCSQTnrTrSnQ==",
"dependencies": {
- "snowtransfer": "^0.8.3"
+ "snowtransfer": "^0.9.0"
},
"engines": {
- "node": ">=12.0.0"
+ "node": ">=14.8.0"
}
},
"node_modules/color": {
@@ -841,6 +839,7 @@
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+ "dev": true,
"dependencies": {
"delayed-stream": "~1.0.0"
},
@@ -1014,6 +1013,7 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+ "dev": true,
"engines": {
"node": ">=0.4.0"
}
@@ -1053,9 +1053,9 @@
}
},
"node_modules/discord-api-types": {
- "version": "0.37.53",
- "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.53.tgz",
- "integrity": "sha512-N6uUgv50OyP981Mfxrrt0uxcqiaNr0BDaQIoqfk+3zM2JpZtwU9v7ce1uaFAP53b2xSDvcbrk80Kneui6XJgGg=="
+ "version": "0.37.60",
+ "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.60.tgz",
+ "integrity": "sha512-5BELXTsv7becqVHrD81nZrqT4oEyXXWBwbsO/kwDDu6X3u19VV1tYDB5I5vaVAK+c1chcDeheI9zACBLm41LiQ=="
},
"node_modules/discord-markdown": {
"version": "2.4.1",
@@ -1345,9 +1345,9 @@
}
},
"node_modules/form-data": {
- "version": "3.0.1",
- "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
- "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+ "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
"dev": true,
"dependencies": {
"asynckit": "^0.4.0",
@@ -1549,9 +1549,9 @@
}
},
"node_modules/heatsync": {
- "version": "2.4.1",
- "resolved": "https://registry.npmjs.org/heatsync/-/heatsync-2.4.1.tgz",
- "integrity": "sha512-cRzLwnKnJ5O4dQWXiJyFp4myKY8lGfK+49/SbPsvnr3pf2PNG1Xh8pPono303cjJeFpaPSTs609mQH1xhPVyzA==",
+ "version": "2.4.2",
+ "resolved": "https://registry.npmjs.org/heatsync/-/heatsync-2.4.2.tgz",
+ "integrity": "sha512-s+YzwGpCjsJLRCuz6Ur8JFKz7vXEMFAPLqDbaEvMR5Um/IPPJpmupBH7LKeiyfGkIScFz9iyBPa1TifcoU4D7A==",
"dependencies": {
"backtracker": "3.3.2"
}
@@ -2192,9 +2192,9 @@
"integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="
},
"node_modules/node-fetch": {
- "version": "2.6.12",
- "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz",
- "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==",
+ "version": "2.7.0",
+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+ "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
"dependencies": {
"whatwg-url": "^5.0.0"
},
@@ -2922,29 +2922,15 @@
}
},
"node_modules/snowtransfer": {
- "version": "0.8.3",
- "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.8.3.tgz",
- "integrity": "sha512-0X6NLFBUKppYT5VH/mVQNGX+ufv0AndunZC84MqGAR/3rfTIGQblgGJlHlDQbeCytlXdMpgRHIGQnBFlE094NQ==",
+ "version": "0.9.0",
+ "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.9.0.tgz",
+ "integrity": "sha512-43Q0pvk7ZV8uZwcL/IhEFYKFZj53FOqxr2dVDwduPT87eHOJzfs8aQ+tNDqsjW6OMUBurwR3XZZFEpQ2f/XzXA==",
"dependencies": {
- "discord-api-types": "^0.37.47",
- "form-data": "^4.0.0",
- "undici": "^5.22.1"
+ "discord-api-types": "^0.37.60",
+ "undici": "^5.26.3"
},
"engines": {
- "node": ">=12.0.0"
- }
- },
- "node_modules/snowtransfer/node_modules/form-data": {
- "version": "4.0.0",
- "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
- "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
- "dependencies": {
- "asynckit": "^0.4.0",
- "combined-stream": "^1.0.8",
- "mime-types": "^2.1.12"
- },
- "engines": {
- "node": ">= 6"
+ "node": ">=14.18.0"
}
},
"node_modules/source-map": {
@@ -3022,14 +3008,6 @@
"node": ">=10"
}
},
- "node_modules/streamsearch": {
- "version": "1.1.0",
- "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
- "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
- "engines": {
- "node": ">=10.0.0"
- }
- },
"node_modules/streamx": {
"version": "2.15.1",
"resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz",
@@ -3122,18 +3100,18 @@
}
},
"node_modules/supertape": {
- "version": "8.3.0",
- "resolved": "https://registry.npmjs.org/supertape/-/supertape-8.3.0.tgz",
- "integrity": "sha512-dcMylmkr1Mctr5UBCrlvZynuBRuLvlkWJLGXdL/PcI41BERnObO+kV0PeZhH5n6lwVnvK2xfvZyN32WIAPf/tw==",
+ "version": "8.6.0",
+ "resolved": "https://registry.npmjs.org/supertape/-/supertape-8.6.0.tgz",
+ "integrity": "sha512-zvAXZaliVu8qpGRx5KiYQfPZcQD9B361lmRtXb3zyilpHHc0/5ygQ9MfWYZEwXywxHDfve3w8ZukI/NKPT9PyA==",
"dev": true,
"dependencies": {
"@cloudcmd/stub": "^4.0.0",
"@putout/cli-keypress": "^1.0.0",
"@putout/cli-validate-args": "^1.0.1",
- "@supertape/engine-loader": "^1.0.0",
+ "@supertape/engine-loader": "^2.0.0",
"@supertape/formatter-fail": "^3.0.0",
"@supertape/formatter-json-lines": "^2.0.0",
- "@supertape/formatter-progress-bar": "^3.0.0",
+ "@supertape/formatter-progress-bar": "^4.0.0",
"@supertape/formatter-short": "^2.0.0",
"@supertape/formatter-tap": "^3.0.0",
"@supertape/operator-stub": "^3.0.0",
@@ -3414,11 +3392,11 @@
}
},
"node_modules/undici": {
- "version": "5.22.1",
- "resolved": "https://registry.npmjs.org/undici/-/undici-5.22.1.tgz",
- "integrity": "sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw==",
+ "version": "5.26.3",
+ "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.3.tgz",
+ "integrity": "sha512-H7n2zmKEWgOllKkIUkLvFmsJQj062lSm3uA4EYApG8gLuiOM0/go9bIoC3HVaSnfg4xunowDE2i9p8drkXuvDw==",
"dependencies": {
- "busboy": "^1.6.0"
+ "@fastify/busboy": "^2.0.0"
},
"engines": {
"node": ">=14.0"
diff --git a/package.json b/package.json
index 6a9deea..29f6f1c 100644
--- a/package.json
+++ b/package.json
@@ -16,9 +16,9 @@
"license": "AGPL-3.0-or-later",
"dependencies": {
"@chriscdn/promise-semaphore": "^2.0.1",
- "better-sqlite3": "^8.3.0",
+ "better-sqlite3": "^9.0.0",
"chunk-text": "^2.0.1",
- "cloudstorm": "^0.8.0",
+ "cloudstorm": "^0.9.0",
"discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3",
"entities": "^4.5.0",
"giframe": "github:cloudrac3r/giframe#v0.4.1",
@@ -31,7 +31,7 @@
"pngjs": "^7.0.0",
"prettier-bytes": "^1.0.4",
"sharp": "^0.32.6",
- "snowtransfer": "^0.8.0",
+ "snowtransfer": "^0.9.0",
"stream-mime-type": "^1.0.2",
"try-to-catch": "^3.0.1",
"turndown": "^7.1.2",
@@ -42,7 +42,7 @@
"@types/node-fetch": "^2.6.3",
"c8": "^8.0.1",
"cross-env": "^7.0.3",
- "discord-api-types": "^0.37.53",
+ "discord-api-types": "^0.37.60",
"supertape": "^8.3.0",
"tap-dot": "github:cloudrac3r/tap-dot#9dd7750ececeae3a96afba91905be812b6b2cc2d"
},
diff --git a/readme.md b/readme.md
index 5c97745..61d1951 100644
--- a/readme.md
+++ b/readme.md
@@ -73,7 +73,7 @@ You'll need:
Follow these steps:
-1. [Get Node.js version 18 or later](https://nodejs.org/en/download/releases) (the version is required by the matrix-appservice dependency)
+1. [Get Node.js version 18 or later](https://nodejs.org/en/download/releases) (the version is required by the better-sqlite3 and matrix-appservice dependencies)
1. Clone this repo and checkout a specific tag. (Development happens on main. Stabler versions are tagged.)
From 9c3f1abd3a2c12d30b57bdec55d85ad352a5dab5 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Sun, 15 Oct 2023 00:26:52 +1300
Subject: [PATCH 029/480] Upload files to Discord as streams for speed
---
m2d/actions/channel-webhook.js | 5 ++--
m2d/actions/send-event.js | 40 +++++++++++++++---------------
m2d/converters/event-to-message.js | 5 ++--
3 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/m2d/actions/channel-webhook.js b/m2d/actions/channel-webhook.js
index 3bb728d..52b4095 100644
--- a/m2d/actions/channel-webhook.js
+++ b/m2d/actions/channel-webhook.js
@@ -2,6 +2,7 @@
const assert = require("assert").strict
const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
const passthrough = require("../../passthrough")
const {discord, db, select} = passthrough
@@ -51,7 +52,7 @@ async function withWebhook(channelID, callback) {
/**
* @param {string} channelID
- * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[]}} data
+ * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]}} data
* @param {string} [threadID]
*/
async function sendMessageWithWebhook(channelID, data, threadID) {
@@ -64,7 +65,7 @@ async function sendMessageWithWebhook(channelID, data, threadID) {
/**
* @param {string} channelID
* @param {string} messageID
- * @param {DiscordTypes.RESTPatchAPIWebhookWithTokenMessageJSONBody & {files?: {name: string, file: Buffer}[]}} data
+ * @param {DiscordTypes.RESTPatchAPIWebhookWithTokenMessageJSONBody & {files?: {name: string, file: Buffer | Readable}[]}} data
* @param {string} [threadID]
*/
async function editMessageWithWebhook(channelID, messageID, data, threadID) {
diff --git a/m2d/actions/send-event.js b/m2d/actions/send-event.js
index 13b5650..9b3fbec 100644
--- a/m2d/actions/send-event.js
+++ b/m2d/actions/send-event.js
@@ -1,11 +1,11 @@
// @ts-check
-const assert = require("assert").strict
-const crypto = require("crypto")
-const {pipeline} = require("stream")
-const {promisify} = require("util")
const Ty = require("../../types")
const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
+const assert = require("assert").strict
+const crypto = require("crypto")
+const fetch = require("node-fetch").default
const passthrough = require("../../passthrough")
const {sync, discord, db, select} = passthrough
@@ -17,13 +17,12 @@ const eventToMessage = sync.require("../converters/event-to-message")
const api = sync.require("../../matrix/api")
/**
- * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer})[]}} message
- * @returns {Promise}
+ * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer | Readable})[]}} message
+ * @returns {Promise}
*/
async function resolvePendingFiles(message) {
if (!message.pendingFiles) return message
const files = await Promise.all(message.pendingFiles.map(async p => {
- let fileBuffer
if ("buffer" in p) {
return {
name: p.name,
@@ -31,21 +30,22 @@ async function resolvePendingFiles(message) {
}
}
if ("key" in p) {
- // Encrypted
+ // Encrypted file
const d = crypto.createDecipheriv("aes-256-ctr", Buffer.from(p.key, "base64url"), Buffer.from(p.iv, "base64url"))
- fileBuffer = await fetch(p.url).then(res => res.arrayBuffer()).then(x => {
- return Buffer.concat([
- d.update(Buffer.from(x)),
- d.final()
- ])
- })
+ // @ts-ignore
+ fetch(p.url).then(res => res.body.pipe(d))
+ return {
+ name: p.name,
+ file: d
+ }
} else {
- // Unencrypted
- fileBuffer = await fetch(p.url).then(res => res.arrayBuffer()).then(x => Buffer.from(x))
- }
- return {
- name: p.name,
- file: fileBuffer // TODO: Once SnowTransfer supports ReadableStreams for attachment uploads, pass in those instead of Buffers
+ // Unencrypted file
+ /** @type {Readable} */ // @ts-ignore
+ const body = await fetch(p.url).then(res => res.body)
+ return {
+ name: p.name,
+ file: body
+ }
}
}))
const newMessage = {
diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js
index 5f6f3e6..8907508 100644
--- a/m2d/converters/event-to-message.js
+++ b/m2d/converters/event-to-message.js
@@ -2,6 +2,7 @@
const Ty = require("../../types")
const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
const chunk = require("chunk-text")
const TurndownService = require("turndown")
const assert = require("assert").strict
@@ -9,8 +10,6 @@ const entities = require("entities")
const passthrough = require("../../passthrough")
const {sync, db, discord, select, from} = passthrough
-/** @type {import("../../matrix/file")} */
-const file = sync.require("../../matrix/file")
/** @type {import("../converters/utils")} */
const utils = sync.require("../converters/utils")
/** @type {import("./emoji-sheet")} */
@@ -245,7 +244,7 @@ async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles)
* @param {{api: import("../../matrix/api")}} di simple-as-nails dependency injection for the matrix API
*/
async function eventToMessage(event, guild, di) {
- /** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[]})[]} */
+ /** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */
let messages = []
let displayName = event.sender
From a542bbdca79ba0825e08fe589f85224e212c9b88 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Mon, 16 Oct 2023 16:24:48 +1300
Subject: [PATCH 030/480] Update discord-markdown to support list and header
---
package-lock.json | 6 +++---
package.json | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/package-lock.json b/package-lock.json
index a005645..11585f8 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -13,7 +13,7 @@
"better-sqlite3": "^9.0.0",
"chunk-text": "^2.0.1",
"cloudstorm": "^0.9.0",
- "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3",
+ "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8",
"entities": "^4.5.0",
"giframe": "github:cloudrac3r/giframe#v0.4.1",
"heatsync": "^2.4.1",
@@ -1058,8 +1058,8 @@
"integrity": "sha512-5BELXTsv7becqVHrD81nZrqT4oEyXXWBwbsO/kwDDu6X3u19VV1tYDB5I5vaVAK+c1chcDeheI9zACBLm41LiQ=="
},
"node_modules/discord-markdown": {
- "version": "2.4.1",
- "resolved": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3",
+ "version": "2.5.1",
+ "resolved": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8",
"license": "MIT",
"dependencies": {
"simple-markdown": "^0.7.2"
diff --git a/package.json b/package.json
index 29f6f1c..42988c8 100644
--- a/package.json
+++ b/package.json
@@ -19,7 +19,7 @@
"better-sqlite3": "^9.0.0",
"chunk-text": "^2.0.1",
"cloudstorm": "^0.9.0",
- "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3",
+ "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8",
"entities": "^4.5.0",
"giframe": "github:cloudrac3r/giframe#v0.4.1",
"heatsync": "^2.4.1",
From 762e48230c682fb9256119a51d600dc6bbafbadf Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Mon, 16 Oct 2023 16:47:42 +1300
Subject: [PATCH 031/480] Ensure the appservice bot user is registered
Synapse does it automatically, but Conduit requires this HTTP call.
---
d2m/actions/register-user.js | 2 +-
scripts/seed.js | 17 ++++++++++++-----
2 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/d2m/actions/register-user.js b/d2m/actions/register-user.js
index 9b5527f..b605c3a 100644
--- a/d2m/actions/register-user.js
+++ b/d2m/actions/register-user.js
@@ -36,7 +36,7 @@ async function createSim(user) {
await api.register(localpart)
} catch (e) {
// If user creation fails, manually undo the database change. Still isn't perfect, but should help.
- // (A transaction would be preferable, but I don't think it's safe to leave transaction open across event loop ticks.)
+ // (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.)
db.prepare("DELETE FROM sim WHERE user_id = ?").run(user.id)
throw e
}
diff --git a/scripts/seed.js b/scripts/seed.js
index 0f2f23d..4fc704c 100644
--- a/scripts/seed.js
+++ b/scripts/seed.js
@@ -67,6 +67,18 @@ async function uploadAutoEmoji(guild, name, filename) {
console.log("✅ Database is ready...")
+ // ensure appservice bot user is registered...
+ try {
+ await api.register(reg.sender_localpart)
+ } catch (e) {
+ if (e.data?.error !== "Internal server error") throw e // "Internal server error" is the only OK error because Synapse says this if you try to register the same username twice.
+ }
+
+ // upload initial images...
+ const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png")
+
+ console.log("✅ Matrix appservice login works...")
+
// upload the L1 L2 emojis to some guild
const emojis = db.prepare("SELECT name FROM auto_emoji WHERE name = 'L1' OR name = 'L2'").pluck().all()
if (emojis.length !== 2) {
@@ -104,11 +116,6 @@ async function uploadAutoEmoji(guild, name, filename) {
}
console.log("✅ Emojis are ready...")
- // ensure homeserver well-known is valid and returns reg.ooye.server_name...
-
- // upload initial images...
- const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png")
-
// set profile data on discord...
const avatarImageBuffer = await fetch("https://cadence.moe/friends/out_of_your_element.png").then(res => res.arrayBuffer())
await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + Buffer.from(avatarImageBuffer).toString("base64")})
From afbbe0da3d9bf561e3f5394c0fb6009a5d9f6711 Mon Sep 17 00:00:00 2001
From: Cadence Ember
Date: Sat, 28 Oct 2023 00:24:42 +1300
Subject: [PATCH 032/480] Fix more edge-case embed formatting
---
.../message-to-event.embeds.test.js | 68 ++++++--
d2m/converters/message-to-event.js | 149 ++++++++++++------
m2d/converters/emoji.js | 1 +
m2d/converters/event-to-message.js | 18 +++
m2d/converters/event-to-message.test.js | 43 ++++-
m2d/converters/utils.js | 65 ++++++++
matrix/matrix-command-handler.js | 53 +------
package-lock.json | 23 ++-
package.json | 4 +-
readme.md | 8 +-
test/data.js | 108 +++++++++++++
test/ooye-test-data.sql | 15 +-
12 files changed, 428 insertions(+), 127 deletions(-)
diff --git a/d2m/converters/message-to-event.embeds.test.js b/d2m/converters/message-to-event.embeds.test.js
index 93d189c..173e016 100644
--- a/d2m/converters/message-to-event.embeds.test.js
+++ b/d2m/converters/message-to-event.embeds.test.js
@@ -35,14 +35,14 @@ test("message2event embeds: nothing but a field", async t => {
$type: "m.room.message",
"m.mentions": {},
msgtype: "m.notice",
- body: "> **Amanda 🎵#2192 :online:"
- + "\n> willow tree, branch 0**"
+ body: "> ### Amanda 🎵#2192 :online:"
+ + "\n> willow tree, branch 0"
+ "\n> **❯ Uptime:**\n> 3m 55s\n> **❯ Memory:**\n> 64.45MB",
format: "org.matrix.custom.html",
- formatted_body: 'Amanda 🎵#2192
'
+ formatted_body: 'Amanda 🎵#2192
'
+ '
willow tree, branch 0'
+ '
❯ Uptime:
3m 55s'
- + '
❯ Memory:
64.45MB
'
+ + '
❯ Memory:
64.45MB
'
}])
})
@@ -52,19 +52,19 @@ test("message2event embeds: reply with just an embed", async t => {
$type: "m.room.message",
msgtype: "m.notice",
"m.mentions": {},
- body: "> [**⏺️ dynastic (@dynastic)**](https://twitter.com/i/user/719631291747078145)"
- + "\n> \n> **https://twitter.com/i/status/1707484191963648161**"
+ body: "> ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145"
+ + "\n> \n> ## https://twitter.com/i/status/1707484191963648161"
+ "\n> \n> does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?"
- + "\n> \n> **Retweets**"
+ + "\n> \n> ### Retweets"
+ "\n> 119"
- + "\n> \n> **Likes**"
+ + "\n> \n> ### Likes"
+ "\n> 5581"
- + "\n> \n> — Twitter",
+ + "\n> — Twitter",
format: "org.matrix.custom.html",
- formatted_body: '⏺️ dynastic (@dynastic)'
- + '
https://twitter.com/i/status/1707484191963648161'
- + '
does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?'
- + '
Retweets
119
Likes
5581
— Twitter
'
+ formatted_body: '⏺️ dynastic (@dynastic)
'
+ + 'https://twitter.com/i/status/1707484191963648161'
+ + '
does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?'
+ + '
Retweets
119
Likes
5581
— Twitter
'
}])
})
@@ -99,3 +99,45 @@ test("message2event embeds: image embed and attachment", async t => {
"m.mentions": {}
}])
})
+
+test("message2event embeds: blockquote in embed", async t => {
+ const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general)
+ t.deepEqual(events, [{
+ $type: "m.room.message",
+ msgtype: "m.text",
+ body: ":emoji: **4 |** #wonderland",
+ format: "org.matrix.custom.html",
+ formatted_body: `
4 | #wonderland`,
+ "m.mentions": {}
+ }, {
+ $type: "m.room.message",
+ msgtype: "m.notice",
+ body: "> ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo\n> \n> reply draft\n> > The following is a message composed via consensus of the Stinker Council.\n> > \n> > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n> > \n> > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n> > \n> > There will be no further communication.\n> \n> [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo)",
+ format: "org.matrix.custom.html",
+ formatted_body: "⏺️ minimus
reply draft
The following is a message composed via consensus of the Stinker Council.
For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.
Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.
There will be no further communication.
Go to Message
",
+ "m.mentions": {}
+ }])
+})
+
+test("message2event embeds: crazy html is all escaped", async t => {
+ const events = await messageToEvent(data.message_with_embeds.escaping_crazy_html_tags, data.guild.general)
+ t.deepEqual(events, [{
+ $type: "m.room.message",
+ msgtype: "m.notice",
+ body: "> ## ⏺️ [Hey