From 024ff34cca0b6a6e234a696234724a778363c279 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 28 Nov 2023 19:04:08 +1300 Subject: [PATCH 001/431] Handle mentioning sims from a different bridge --- m2d/converters/event-to-message.js | 17 ++++++++++++---- m2d/converters/event-to-message.test.js | 27 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index cf51d86..ec84e07 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -390,10 +390,19 @@ async function eventToMessage(event, guild, di) { // Handling mentions of Discord users input = input.replace(/("https:\/\/matrix.to\/#\/(@[^"]+)")>/g, (whole, attributeValue, mxid) => { - if (!utils.eventSenderIsFromDiscord(mxid)) return whole - const userID = select("sim", "user_id", {mxid: mxid}).pluck().get() - if (!userID) return whole - return `${attributeValue} data-user-id="${userID}">` + if (utils.eventSenderIsFromDiscord(mxid)) { + // Handle mention of an OOYE sim user by their mxid + const userID = select("sim", "user_id", {mxid: mxid}).pluck().get() + if (!userID) return whole + return `${attributeValue} data-user-id="${userID}">` + } else { + // Handle mention of a Matrix user by their mxid + // Check if this Matrix user is actually the sim user from another old bridge in the room? + const match = mxid.match(/[^:]*discord[^:]*_([0-9]{6,}):/) // try to match @_discord_123456, @_discordpuppet_123456, etc. + if (match) return `${attributeValue} data-user-id="${match[1]}">` + // Nope, just a real Matrix user. + return whole + } }) // Handling mentions of Discord rooms diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 40eb83e..6dbe112 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -2211,6 +2211,33 @@ test("event2message: guessed @mentions may join members to mention", async t => t.equal(called, 1, "searchGuildMembers should be called once") }) +test("event2message: guessed @mentions work with other matrix bridge old users", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "extremity#0: zenosia#0717: back me up on this sentiment, if not necessarily the phrasing", + format: "org.matrix.custom.html", + formatted_body: "extremity#0: zenosia#0717: back me up on this sentiment, if not necessarily the phrasing" + }, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + event_id: "$SiXetU9h9Dg-M9Frcw_C6ahnoXZ3QPZe3MVJR5tcB9A" + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "<@114147806469554185> <@176943908762006200> back me up on this sentiment, if not necessarily the phrasing", + avatar_url: undefined + }], + ensureJoined: [] // we already think it worked on Matrix side due to the pill, so no need for the OOYE sim user to join the room to indicate success. + } + ) +}) + slow()("event2message: unknown emoji in the end is reuploaded as a sprite sheet", async t => { const messages = await eventToMessage({ type: "m.room.message", From 2df7c665cb67a40f8b5573f53fd6729a762a9543 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 30 Nov 2023 16:27:40 +1300 Subject: [PATCH 002/431] m->d: Message links are now guessed when unknown --- d2m/converters/message-to-event.js | 9 +- m2d/converters/event-to-message.js | 59 ++++++++++--- m2d/converters/event-to-message.test.js | 108 +++++++++++++++++++++++- test/test.js | 7 +- 4 files changed, 164 insertions(+), 19 deletions(-) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 92bc241..2026e07 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -269,12 +269,12 @@ async function messageToEvent(message, guild, options = {}, di) { */ async function transformContentMessageLinks(content) { let offset = 0 - for (const match of [...content.matchAll(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/([0-9]+)\/([0-9]+)\/([0-9]+)/g)]) { + for (const match of [...content.matchAll(/https:\/\/(?:ptb\.|canary\.|www\.)?discord(?:app)?\.com\/channels\/[0-9]+\/([0-9]+)\/([0-9]+)/g)]) { assert(typeof match.index === "number") - const channelID = match[2] - const messageID = match[3] - const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() + const [_, channelID, messageID] = match let result + + const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() if (roomID) { const eventID = select("event_message", "event_id", {message_id: messageID}).pluck().get() if (eventID && roomID) { @@ -287,6 +287,7 @@ async function messageToEvent(message, guild, options = {}, di) { } else { result = `${match[0]} [event is from another server]` } + content = content.slice(0, match.index + offset) + result + content.slice(match.index + match[0].length + offset) offset += result.length - match[0].length } diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index ec84e07..f7053cd 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -11,7 +11,9 @@ const entities = require("entities") const passthrough = require("../../passthrough") const {sync, db, discord, select, from} = passthrough /** @type {import("../converters/utils")} */ -const utils = sync.require("../converters/utils") +const mxUtils = sync.require("../converters/utils") +/** @type {import("../../discord/utils")} */ +const dUtils = sync.require("../../discord/utils") /** @type {import("./emoji-sheet")} */ const emojiSheet = sync.require("./emoji-sheet") @@ -102,6 +104,7 @@ turndownService.addRule("inlineLink", { replacement: function (content, node) { if (node.getAttribute("data-user-id")) return `<@${node.getAttribute("data-user-id")}>` + if (node.getAttribute("data-message-id")) return `https://discord.com/channels/${node.getAttribute("data-guild-id")}/${node.getAttribute("data-channel-id")}/${node.getAttribute("data-message-id")}` if (node.getAttribute("data-channel-id")) return `<#${node.getAttribute("data-channel-id")}>` const href = node.getAttribute("href") let brackets = ["", ""] @@ -162,7 +165,7 @@ turndownService.addRule("emoji", { return `<::>` } else { // We prefer not to upload this as a sprite sheet because the emoji is not at the end of the message, it is in the middle. - return `[${node.getAttribute("title")}](${utils.getPublicUrlForMxc(mxcUrl)})` + return `[${node.getAttribute("title")}](${mxUtils.getPublicUrlForMxc(mxcUrl)})` } } }) @@ -276,7 +279,7 @@ async function eventToMessage(event, guild, di) { // Try to extract an accurate display name and avatar URL from the member event const member = await getMemberFromCacheOrHomeserver(event.room_id, event.sender, di?.api) if (member.displayname) displayName = member.displayname - if (member.avatar_url) avatarURL = utils.getPublicUrlForMxc(member.avatar_url) || undefined + if (member.avatar_url) avatarURL = mxUtils.getPublicUrlForMxc(member.avatar_url) || undefined // If the display name is too long to be put into the webhook (80 characters is the maximum), // put the excess characters into displayNameRunoff, later to be put at the top of the message let [displayNameShortened, displayNameRunoff] = splitDisplayName(displayName) @@ -390,7 +393,7 @@ async function eventToMessage(event, guild, di) { // Handling mentions of Discord users input = input.replace(/("https:\/\/matrix.to\/#\/(@[^"]+)")>/g, (whole, attributeValue, mxid) => { - if (utils.eventSenderIsFromDiscord(mxid)) { + if (mxUtils.eventSenderIsFromDiscord(mxid)) { // Handle mention of an OOYE sim user by their mxid const userID = select("sim", "user_id", {mxid: mxid}).pluck().get() if (!userID) return whole @@ -405,12 +408,42 @@ async function eventToMessage(event, guild, di) { } }) - // Handling mentions of Discord rooms - input = input.replace(/("https:\/\/matrix.to\/#\/(![^"]+)")>/g, (whole, attributeValue, roomID) => { + // Handling mentions of rooms and room-messages + let offset = 0 + for (const match of [...input.matchAll(/("https:\/\/matrix.to\/#\/(![^"/?]+)(?:\/(\$[^"/?]+))?(?:\?[^"]*)?")>/g)]) { + assert(typeof match.index === "number") + const [_, attributeValue, roomID, eventID] = match + let result + + // Don't process links that are part of the reply fallback, they'll be removed entirely by turndown + if (input.slice(match.index + match[0].length + offset).startsWith("In reply to")) continue + const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get() - if (!channelID) return whole - return `${attributeValue} data-channel-id="${channelID}">` - }) + if (!channelID) continue + if (!eventID) { + // 1: It's a room link, so <#link> to the channel + result = `${attributeValue} data-channel-id="${channelID}">` + } else { + // Linking to a particular event with a discord.com/channels/guildID/channelID/messageID link + // Need to know the guildID and messageID + const guildID = discord.channels.get(channelID)?.["guild_id"] + if (!guildID) continue + const messageID = select("event_message", "message_id", {event_id: eventID}).pluck().get() + if (messageID) { + // 2: Linking to a known event + result = `${attributeValue} data-channel-id="${channelID}" data-guild-id="${guildID}" data-message-id="${messageID}">` + } else { + // 3: Linking to an unknown event that OOYE didn't originally bridge - we can guess messageID from the timestamp + const originalEvent = await di.api.getEvent(roomID, eventID) + if (!originalEvent) continue + const guessedMessageID = dUtils.timestampToSnowflakeInexact(originalEvent.origin_server_ts) + result = `${attributeValue} data-channel-id="${channelID}" data-guild-id="${guildID}" data-message-id="${guessedMessageID}">` + } + } + + input = input.slice(0, match.index + offset) + result + input.slice(match.index + match[0].length + offset) + offset += result.length - match[0].length + } // Stripping colons after mentions input = input.replace(/( data-user-id.*?<\/a>):?/g, "$1") @@ -430,7 +463,7 @@ async function eventToMessage(event, guild, di) { beforeTag = beforeTag || "" afterContext = afterContext || "" afterTag = afterTag || "" - if (!utils.BLOCK_ELEMENTS.includes(beforeTag.toUpperCase()) && !utils.BLOCK_ELEMENTS.includes(afterTag.toUpperCase())) { + if (!mxUtils.BLOCK_ELEMENTS.includes(beforeTag.toUpperCase()) && !mxUtils.BLOCK_ELEMENTS.includes(afterTag.toUpperCase())) { return beforeContext + "
" + afterContext } else { return whole @@ -480,13 +513,13 @@ async function eventToMessage(event, guild, di) { const filename = event.content.body if ("url" in event.content) { // Unencrypted - const url = utils.getPublicUrlForMxc(event.content.url) + const url = mxUtils.getPublicUrlForMxc(event.content.url) assert(url) attachments.push({id: "0", filename}) pendingFiles.push({name: filename, url}) } else { // Encrypted - const url = utils.getPublicUrlForMxc(event.content.file.url) + const url = mxUtils.getPublicUrlForMxc(event.content.file.url) assert(url) assert.equal(event.content.file.key.alg, "A256CTR") attachments.push({id: "0", filename}) @@ -494,7 +527,7 @@ async function eventToMessage(event, guild, di) { } } else if (event.type === "m.sticker") { content = "" - const url = utils.getPublicUrlForMxc(event.content.url) + const url = mxUtils.getPublicUrlForMxc(event.content.url) assert(url) let filename = event.content.body if (event.type === "m.sticker") { diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 6dbe112..9d2c70c 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -1701,7 +1701,7 @@ test("event2message: mentioning bridged rooms works", async t => { msgtype: "m.text", body: "wrong body", format: "org.matrix.custom.html", - formatted_body: `I'm just worm-form testing channel mentions` + formatted_body: `I'm just worm-farm testing channel mentions` }, event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", origin_server_ts: 1688301929913, @@ -1725,6 +1725,112 @@ test("event2message: mentioning bridged rooms works", async t => { ) }) +test("event2message: mentioning known bridged events works", async t => { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: `it was uploaded earlier in amanda-spam` + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "it was uploaded earlier in https://discord.com/channels/497159726455455754/497161350934560778/1141619794500649020", + avatar_url: undefined + }] + } + ) +}) + +test("event2message: mentioning unknown bridged events works", async t => { + let called = 0 + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: `it was uploaded years ago in amanda-spam` + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }, {}, { + api: { + async getEvent(roomID, eventID) { + called++ + t.equal(roomID, "!CzvdIdUQXgUjDVKxeU:cadence.moe") + t.equal(eventID, "$zpzx6ABetMl8BrpsFbdZ7AefVU1Y_-t97bJRJM2JyW0") + return { + origin_server_ts: 1599813121000 + } + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "it was uploaded years ago in https://discord.com/channels/497159726455455754/497161350934560778/753895613661184000", + avatar_url: undefined + }] + } + ) + t.equal(called, 1, "getEvent should be called once") +}) + +test("event2message: link to event in an unknown room", async t => { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: 'ah yeah, here\'s where the bug was reported: https://matrix.to/#/!QtykxKocfZaZOUrTwp:matrix.org/$1542477546853947KGhZL:matrix.org' + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "ah yeah, here's where the bug was reported: [https://matrix.to/#/!QtykxKocfZaZOUrTwp:matrix.org/$1542477546853947KGhZL:matrix.org]()", + avatar_url: undefined + }] + } + ) +}) + test("event2message: colon after mentions is stripped", async t => { t.deepEqual( await eventToMessage({ diff --git a/test/test.js b/test/test.js index 553ec44..4663b18 100644 --- a/test/test.js +++ b/test/test.js @@ -24,7 +24,12 @@ const discord = { ]), application: { id: "684280192553844747" - } + }, + channels: new Map([ + ["497161350934560778", { + guild_id: "497159726455455754" + }] + ]) } Object.assign(passthrough, { discord, config, sync, db }) From 4dcdd0287ea129cba0c8b3d5d675f9692733f293 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 30 Nov 2023 17:10:41 +1300 Subject: [PATCH 003/431] Check for M_USER_IN_USE in seed.js --- scripts/seed.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/seed.js b/scripts/seed.js index 4fc704c..4dc08e0 100644 --- a/scripts/seed.js +++ b/scripts/seed.js @@ -71,7 +71,7 @@ async function uploadAutoEmoji(guild, name, filename) { try { await api.register(reg.sender_localpart) } catch (e) { - if (e.data?.error !== "Internal server error") throw e // "Internal server error" is the only OK error because Synapse says this if you try to register the same username twice. + if (e.errcode === "M_USER_IN_USE" || e.data?.error !== "Internal server error") throw e // "Internal server error" is the only OK error because Synapse says this if you try to register the same username twice. } // upload initial images... From c8742f95124402fcfa4816966cbebbeaf3d5252b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 2 Dec 2023 17:13:10 +1300 Subject: [PATCH 004/431] m->d: Fix message ID guess on plaintext events --- m2d/converters/emoji.js | 1 + m2d/converters/event-to-message.js | 94 ++++++++++++++++--------- m2d/converters/event-to-message.test.js | 62 +++++++++++++++- 3 files changed, 121 insertions(+), 36 deletions(-) diff --git a/m2d/converters/emoji.js b/m2d/converters/emoji.js index 19f6a15..bb6a3b0 100644 --- a/m2d/converters/emoji.js +++ b/m2d/converters/emoji.js @@ -41,6 +41,7 @@ function encodeEmoji(input, shortcode) { "%F0%9F%90%88", // 🐈 "%E2%9D%93", // ❓ "%F0%9F%8F%86", // 🏆️ + "%F0%9F%93%9A", // 📚️ ] discordPreferredEncoding = diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index f7053cd..9518bbc 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -259,6 +259,62 @@ async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles) return content } +/** + * @param {string} input + * @param {{api: import("../../matrix/api")}} di simple-as-nails dependency injection for the matrix API + */ +async function handleRoomOrMessageLinks(input, di) { + let offset = 0 + for (const match of [...input.matchAll(/("?https:\/\/matrix.to\/#\/(![^"/, ?)]+)(?:\/(\$[^"/ ?)]+))?(?:\?[^",:!? )]*)?)(">|[, )]|$)/g)]) { + assert(typeof match.index === "number") + const [_, attributeValue, roomID, eventID, endMarker] = match + let result + + const resultType = endMarker === '">' ? "html" : "plain" + const MAKE_RESULT = { + ROOM_LINK: { + html: channelID => `${attributeValue}" data-channel-id="${channelID}">`, + plain: channelID => `<#${channelID}>${endMarker}` + }, + MESSAGE_LINK: { + html: (guildID, channelID, messageID) => `${attributeValue}" data-channel-id="${channelID}" data-guild-id="${guildID}" data-message-id="${messageID}">`, + plain: (guildID, channelID, messageID) => `https://discord.com/channels/${guildID}/${channelID}/${messageID}${endMarker}` + } + } + + // Don't process links that are part of the reply fallback, they'll be removed entirely by turndown + if (input.slice(match.index + match[0].length + offset).startsWith("In reply to")) continue + + const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get() + if (!channelID) continue + if (!eventID) { + // 1: It's a room link, so <#link> to the channel + result = MAKE_RESULT.ROOM_LINK[resultType](channelID) + } else { + // Linking to a particular event with a discord.com/channels/guildID/channelID/messageID link + // Need to know the guildID and messageID + const guildID = discord.channels.get(channelID)?.["guild_id"] + if (!guildID) continue + const messageID = select("event_message", "message_id", {event_id: eventID}).pluck().get() + if (messageID) { + // 2: Linking to a known event + result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, messageID) + } else { + // 3: Linking to an unknown event that OOYE didn't originally bridge - we can guess messageID from the timestamp + const originalEvent = await di.api.getEvent(roomID, eventID) + if (!originalEvent) continue + const guessedMessageID = dUtils.timestampToSnowflakeInexact(originalEvent.origin_server_ts) + result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, guessedMessageID) + } + } + + input = input.slice(0, match.index + offset) + result + input.slice(match.index + match[0].length + offset) + offset += result.length - match[0].length + } + + return input +} + /** * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event * @param {import("discord-api-types/v10").APIGuild} guild @@ -409,41 +465,7 @@ async function eventToMessage(event, guild, di) { }) // Handling mentions of rooms and room-messages - let offset = 0 - for (const match of [...input.matchAll(/("https:\/\/matrix.to\/#\/(![^"/?]+)(?:\/(\$[^"/?]+))?(?:\?[^"]*)?")>/g)]) { - assert(typeof match.index === "number") - const [_, attributeValue, roomID, eventID] = match - let result - - // Don't process links that are part of the reply fallback, they'll be removed entirely by turndown - if (input.slice(match.index + match[0].length + offset).startsWith("In reply to")) continue - - const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get() - if (!channelID) continue - if (!eventID) { - // 1: It's a room link, so <#link> to the channel - result = `${attributeValue} data-channel-id="${channelID}">` - } else { - // Linking to a particular event with a discord.com/channels/guildID/channelID/messageID link - // Need to know the guildID and messageID - const guildID = discord.channels.get(channelID)?.["guild_id"] - if (!guildID) continue - const messageID = select("event_message", "message_id", {event_id: eventID}).pluck().get() - if (messageID) { - // 2: Linking to a known event - result = `${attributeValue} data-channel-id="${channelID}" data-guild-id="${guildID}" data-message-id="${messageID}">` - } else { - // 3: Linking to an unknown event that OOYE didn't originally bridge - we can guess messageID from the timestamp - const originalEvent = await di.api.getEvent(roomID, eventID) - if (!originalEvent) continue - const guessedMessageID = dUtils.timestampToSnowflakeInexact(originalEvent.origin_server_ts) - result = `${attributeValue} data-channel-id="${channelID}" data-guild-id="${guildID}" data-message-id="${guessedMessageID}">` - } - } - - input = input.slice(0, match.index + offset) + result + input.slice(match.index + match[0].length + offset) - offset += result.length - match[0].length - } + input = await handleRoomOrMessageLinks(input, di) // Stripping colons after mentions input = input.replace(/( data-user-id.*?<\/a>):?/g, "$1") @@ -504,6 +526,8 @@ async function eventToMessage(event, guild, di) { content = `* ${displayName} ${content}` } + content = await handleRoomOrMessageLinks(content, di) + // Markdown needs to be escaped, though take care not to escape the middle of links // @ts-ignore bad type from turndown content = turndownService.escape(content) diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 9d2c70c..e3a6905 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -1725,7 +1725,67 @@ test("event2message: mentioning bridged rooms works", async t => { ) }) -test("event2message: mentioning known bridged events works", async t => { +test("event2message: mentioning known bridged events works (plaintext body)", async t => { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "it was uploaded earlier in https://matrix.to/#/!CzvdIdUQXgUjDVKxeU:cadence.moe/$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10?via=cadence.moe, take a look!" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "it was uploaded earlier in https://discord.com/channels/497159726455455754/497161350934560778/1141619794500649020, take a look!", + avatar_url: undefined + }] + } + ) +}) + +test("event2message: mentioning known bridged events works (partially formatted body)", async t => { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: `it was uploaded earlier in https://matrix.to/#/!CzvdIdUQXgUjDVKxeU:cadence.moe/$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10?via=cadence.moe` + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "it was uploaded earlier in https://discord.com/channels/497159726455455754/497161350934560778/1141619794500649020", + avatar_url: undefined + }] + } + ) +}) + +test("event2message: mentioning known bridged events works (formatted body)", async t => { t.deepEqual( await eventToMessage({ content: { From 3a8dbe8c59c414219fdcd03aa32df306486e7c45 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 15 Dec 2023 17:08:06 +1300 Subject: [PATCH 005/431] Publish WIP PK notes --- docs/pluralkit-notetaking.md | 83 ++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 docs/pluralkit-notetaking.md diff --git a/docs/pluralkit-notetaking.md b/docs/pluralkit-notetaking.md new file mode 100644 index 0000000..9420e17 --- /dev/null +++ b/docs/pluralkit-notetaking.md @@ -0,0 +1,83 @@ +## What is PluralKit + +PluralKit is a Discord bot. After a Discord user registers with PK, PK will delete and repost their messages. The reposted messages will be sent by a webhook with a custom display name and avatar. This effectively lets a person assume a custom display name and avatar at will on a per-message basis. People use this for roleplaying and/or dissociative-identity-disorder things. PK is extremely popular. + +## PK terminology + +- **Proxying:** The act of deleting and reposting messages. +- **Member:** Identity that messages will be posted by. +- **System:** Systems contain members. A system is usually controlled by one Discord account, but it's also possible to have multiple accounts be part of the same system. + +## PK API schema + +https://pluralkit.me/api/models/ + +## Experience on OOYE without special PK handling + +1. Message is sent by Discord user and copied to Matrix-side. +1. The message is immediately deleted by PK and deleted from Matrix-side. +1. The message is resent by the PK webhook and copied to Matrix-side (by @_ooye_bot) with limited authorship information. + +## Experience on Half-Shot's bridge without special PK handling + +1. Message is sent by Discord user and copied to Matrix-side. +1. The message is immediately deleted by PK and deleted from Matrix-side. +1. The message is resent by the PK webhook and copied to Matrix-side _by a dedicated sim user for that webhook's username._ + +If a PK system member changes their display name, the webhook display name will change too. But Half-Shot's bridge can't keep track of webhook message authorship. It uses the webhook's display name to determine whether to reuse the previous sim user account. This makes Half-Shot's bridge create a brand new sim user for the same system member, and causes the Matrix-side member list to eventually fill up with lots of abandoned sim users named @_discord_NUMBERS_NUMBERS_GARBLED_NAME. + +## Goals of special PK handling + +1. Avoid bridging the send-delete-send dance (solution: the speedbump) +2. Attribute message authorship to the actual PK system member (solution: system member mapping) +3. Avoid creating too many sim users (solution: OOYE sending other webhook messages as @_ooye_bot) + +## What is the speedbump (goal 1) + +When a Discord user sends a message, we can't know whether or not it's about to be deleted by PK. + +If PK doesn't plan to delete the message, we should deliver it straight away to Matrix-side. + +But if PK does plan to delete the message, we shouldn't bridge it at all. We should wait until the PK webhook sends the replacement message, then deliver _that_ message to Matrix-side. + +Unfortunately, we can't see into the future. We don't know if PK will delete the message or not. + +My solution is the speedbump. In speedbump-enabled channels, OOYE will wait a few seconds before delivering the message. The **purpose of the speedbump is to avoid the send-delete-send dance** by not bridging a message until we know it's supposed to stay. + +## Configuring the speedbump + +Nuh-uh. Offering configuration creates an opportunity for misconfiguration. OOYE wants to act in the best possible way with the default settings. In general, everything in OOYE should work in an intelligent, predictable way without having to think about it. + +Since it slows down messages, the speedbump has a negative impact on user experience if it's not needed. So OOYE will automatically activate and deactivate the speedbump if it's necessary. Here's how it works. + +When a message is deleted in a channel, the following logic is triggered: + +1. Discord API: Get the list of webhooks in this channel. +1. If there is a webhook owned by PK, speedbump mode is now ON. Otherwise, speedbump mode is now OFF. + +This check is only done every so often to avoid encountering the Discord API's rate limits. + +## PK system member mapping (goal 2) + +PK system members need to be mapped to individual Matrix sim users, so we need to map the member data to all the fields of a Matrix profile. (This will replace the existing logic of `userToSimName`.) I'll map them in this way: + +- **Matrix ID:** `@_ooye_pk_[FIVE_CHAR_ID].example.org` +- **Display name:** `[NAME] [[PRONOUNS]]` +- **Avatar:** webhook_avatar_url ?? avatar_url + +I'll get this data by calling the PK API for each message: https://api.pluralkit.me/v2/messages/[PK_WEBHOOK_MESSAGE_ID] + +## Special code paths for PK users + +TBD + +## Database schema + +TBD + +## Unsolved problems + +- How does message editing work? +- Improve the contents of PK's reply embeds to be the actual reply text, not the OOYE context preamble +- Unused or removed system members should be removed from the member list too. +- When a Discord user leaves a server, all their system members should leave the member list too. (I also have to solve this for regular non-PK users.) From cf25e1661b4b04886b49ae029dcb5c366e7ddcbb Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 21 Dec 2023 08:23:35 +1300 Subject: [PATCH 006/431] Add test data for attachments with description --- test/data.js | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/data.js b/test/data.js index 3e78802..ed62a2d 100644 --- a/test/data.js +++ b/test/data.js @@ -869,6 +869,51 @@ module.exports = { ], guild_id: '1100319549670301727' }, + attachment_with_description: { + id: "1187111292243288194", + type: 0, + content: "", + channel_id: "1100319550446252084", + author: { + id: "772659086046658620", + username: "cadence.worm", + avatar: "4b5c4b28051144e4c111f0113a0f1cf1", + discriminator: "0", + public_flags: 0, + premium_type: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "cadence", + avatar_decoration_data: null, + banner_color: null + }, + attachments: [ + { + id: "1187111291932905554", + filename: "image.png", + size: 50208, + url: "https://cdn.discordapp.com/attachments/1100319550446252084/1187111291932905554/image.png?ex=6595b28b&is=65833d8b&hm=6a3c07328749b9d0d1a612ea0cbf1711a7fe29aeaa833c12a6eb6d5db1a87ea4&", + proxy_url: "https://media.discordapp.net/attachments/1100319550446252084/1187111291932905554/image.png?ex=6595b28b&is=65833d8b&hm=6a3c07328749b9d0d1a612ea0cbf1711a7fe29aeaa833c12a6eb6d5db1a87ea4&", + width: 412, + height: 228, + description: "here is my description!", + content_type: "image/png", + placeholder: "C/gBBIDPqKiim3h8hpBMv8RvVw==", + placeholder_version: 1 + } + ], + embeds: [], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2023-12-20T19:16:27.532000+00:00", + edited_timestamp: null, + flags: 0, + components: [] + }, skull_webp_attachment_with_content: { type: 0, tts: false, From e4f66a50770b6087f6cda69e4cdbb7d6539fcc59 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 21 Dec 2023 16:56:36 +1300 Subject: [PATCH 007/431] Notetaking about how PK code paths will work --- docs/pluralkit-notetaking.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/pluralkit-notetaking.md b/docs/pluralkit-notetaking.md index 9420e17..697cdee 100644 --- a/docs/pluralkit-notetaking.md +++ b/docs/pluralkit-notetaking.md @@ -69,7 +69,19 @@ I'll get this data by calling the PK API for each message: https://api.pluralkit ## Special code paths for PK users -TBD +When a message is deleted, re-evaluate speedbump mode if necessary, and store who the PK webhook is for this channel if exists. + +When a message is received and the speedbump is enabled, put it into a queue to be sent a few seconds later. + +When a message is deleted, remove it from the queue. + +When a message is received, if it's from a webhook, and the webhook is in the "speedbump_webhook" table, and the webhook user ID is the public PK instance, then look up member details in the PK API, and use a different MXID mapping algorithm based on those details. + +### Edits should Just Work without any special code paths + +Proxied messages are edited by sending "pk;edit blah blah" as a reply to the message to edit. PK will delete the edit command and use the webhook edit endpoint to update the message. + +OOYE's speedbump will prevent the edit command appearing at all on Matrix-side, and OOYE already understands how to do webhook edits. ## Database schema @@ -77,7 +89,8 @@ TBD ## Unsolved problems -- How does message editing work? - Improve the contents of PK's reply embeds to be the actual reply text, not the OOYE context preamble +- Possibly change OOYE's reply context to be an embed (for visual consistency with replies from PK users) +- Possibly extract information from OOYE's reply embed and transform it into an mx-reply structure for Matrix users - Unused or removed system members should be removed from the member list too. - When a Discord user leaves a server, all their system members should leave the member list too. (I also have to solve this for regular non-PK users.) From 83070dcf7e50b3e52b0a91c9a597607c8ad67d46 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 6 Jan 2024 19:00:57 +1300 Subject: [PATCH 008/431] Fix translating URL encoded mentions of sim users --- m2d/converters/event-to-message.js | 3 ++- m2d/converters/event-to-message.test.js | 32 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 9518bbc..1afd882 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -448,7 +448,8 @@ async function eventToMessage(event, guild, di) { } // Handling mentions of Discord users - input = input.replace(/("https:\/\/matrix.to\/#\/(@[^"]+)")>/g, (whole, attributeValue, mxid) => { + input = input.replace(/("https:\/\/matrix.to\/#\/((?:@|%40)[^"]+)")>/g, (whole, attributeValue, mxid) => { + mxid = decodeURIComponent(mxid) if (mxUtils.eventSenderIsFromDiscord(mxid)) { // Handle mention of an OOYE sim user by their mxid const userID = select("sim", "user_id", {mxid: mxid}).pluck().get() diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index e3a6905..4b30292 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -1663,6 +1663,38 @@ test("event2message: mentioning discord users works", async t => { ) }) + +test("event2message: mentioning discord users works when URL encoded", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "Crunch God a sample message", + format: "org.matrix.custom.html", + formatted_body: `Crunch God a sample message`, + msgtype: "m.text" + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "<@771520384671416320> a sample message", + avatar_url: undefined + }] + } + ) +}) + test("event2message: mentioning matrix users works", async t => { t.deepEqual( await eventToMessage({ From 60cf40680f079c34a8377b3e562dd3fcada9a8bf Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 6 Jan 2024 19:10:52 +1300 Subject: [PATCH 009/431] d->m: Alt text should be bridged for any file type --- d2m/converters/message-to-event.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 2026e07..60aba95 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -127,7 +127,7 @@ async function attachmentToEvent(mentions, attachment) { msgtype: "m.image", url: await file.uploadDiscordFileToMxc(attachment.url), external_url: attachment.url, - body: attachment.filename, + body: attachment.description || attachment.filename, filename: attachment.filename, info: { mimetype: attachment.content_type, @@ -174,7 +174,7 @@ async function attachmentToEvent(mentions, attachment) { msgtype: "m.file", url: await file.uploadDiscordFileToMxc(attachment.url), external_url: attachment.url, - body: attachment.filename, + body: attachment.description || attachment.filename, filename: attachment.filename, info: { mimetype: attachment.content_type, From 84d791cd8ab858eb99c40c292047ed2ac19cf711 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 6 Jan 2024 19:42:13 +1300 Subject: [PATCH 010/431] m->d: Support attachment body data as alt text --- m2d/converters/event-to-message.js | 9 ++++-- m2d/converters/event-to-message.test.js | 43 +++++++++++++++++++++++-- types.d.ts | 2 ++ 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 1afd882..37262ce 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -535,19 +535,22 @@ async function eventToMessage(event, guild, di) { } } else if (event.type === "m.room.message" && (event.content.msgtype === "m.file" || event.content.msgtype === "m.video" || event.content.msgtype === "m.audio" || event.content.msgtype === "m.image")) { content = "" - const filename = event.content.body + const filename = event.content.filename || event.content.body + // A written `event.content.body` will be bridged to Discord's image `description` which is like alt text. + // Bridging as description rather than message content in order to match Matrix clients (Element, Neochat) which treat this as alt text or title text. + const description = (event.content.body !== event.content.filename && event.content.filename && event.content.body) || undefined if ("url" in event.content) { // Unencrypted const url = mxUtils.getPublicUrlForMxc(event.content.url) assert(url) - attachments.push({id: "0", filename}) + attachments.push({id: "0", description, filename}) pendingFiles.push({name: filename, url}) } else { // Encrypted const url = mxUtils.getPublicUrlForMxc(event.content.file.url) assert(url) assert.equal(event.content.file.key.alg, "A256CTR") - attachments.push({id: "0", filename}) + attachments.push({id: "0", description, filename}) pendingFiles.push({name: filename, url, key: event.content.file.key.k, iv: event.content.file.iv}) } } else if (event.type === "m.sticker") { diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 4b30292..8a57c41 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -2143,7 +2143,7 @@ test("event2message: text attachments work", async t => { username: "cadence [they]", content: "", avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU", - attachments: [{id: "0", filename: "chiki-powerups.txt"}], + attachments: [{id: "0", description: undefined, filename: "chiki-powerups.txt"}], pendingFiles: [{name: "chiki-powerups.txt", url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/zyThGlYQxvlvBVbVgKDDbiHH"}] }] } @@ -2157,6 +2157,7 @@ test("event2message: image attachments work", async t => { sender: "@cadence:cadence.moe", content: { body: "cool cat.png", + filename: "cool cat.png", info: { size: 43170, mimetype: "image/png", @@ -2178,7 +2179,43 @@ test("event2message: image attachments work", async t => { username: "cadence [they]", content: "", avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU", - attachments: [{id: "0", filename: "cool cat.png"}], + attachments: [{id: "0", description: undefined, filename: "cool cat.png"}], + pendingFiles: [{name: "cool cat.png", url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/IvxVJFLEuksCNnbojdSIeEvn"}] + }] + } + ) +}) + +test("event2message: image attachments can have a custom description", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + body: "Cat emoji surrounded by pink hearts", + filename: "cool cat.png", + info: { + size: 43170, + mimetype: "image/png", + w: 480, + h: 480, + "xyz.amorgan.blurhash": "URTHsVaTpdj2eKZgkkkXp{pHl7feo@lSl9Z$" + }, + msgtype: "m.image", + url: "mxc://cadence.moe/IvxVJFLEuksCNnbojdSIeEvn" + }, + event_id: "$CXQy3Wmg1A-gL_xAesC1HQcQTEXwICLdSwwUx55FBTI", + room_id: "!BnKuBPCvyfOkhcUjEu:cadence.moe" + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "", + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU", + attachments: [{id: "0", description: "Cat emoji surrounded by pink hearts", filename: "cool cat.png"}], pendingFiles: [{name: "cool cat.png", url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/IvxVJFLEuksCNnbojdSIeEvn"}] }] } @@ -2228,7 +2265,7 @@ test("event2message: encrypted image attachments work", async t => { username: "cadence [they]", content: "", avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU", - attachments: [{id: "0", filename: "image.png"}], + attachments: [{id: "0", description: undefined, filename: "image.png"}], pendingFiles: [{ name: "image.png", url: "https://matrix.cadence.moe/_matrix/media/r0/download/heyquark.com/LOGkUTlVFrqfiExlGZNgCJJX", diff --git a/types.d.ts b/types.d.ts index e93f379..9e9d72b 100644 --- a/types.d.ts +++ b/types.d.ts @@ -91,6 +91,7 @@ export namespace Event { export type M_Room_Message_File = { msgtype: "m.file" | "m.image" | "m.video" | "m.audio" body: string + filename?: string url: string info?: any "m.relates_to"?: { @@ -107,6 +108,7 @@ export namespace Event { export type M_Room_Message_Encrypted_File = { msgtype: "m.file" | "m.image" | "m.video" | "m.audio" body: string + filename?: string file: { url: string iv: string From 8e3b674d9066d37ed9673cfa36bcdad0b5b2ff8d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 10 Jan 2024 15:48:13 +1300 Subject: [PATCH 011/431] Forget messages/events upon deletion (may fix #18) --- d2m/actions/delete-message.js | 3 ++- m2d/actions/redact.js | 2 ++ m2d/actions/send-event.js | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/d2m/actions/delete-message.js b/d2m/actions/delete-message.js index 30c31d4..f893e6d 100644 --- a/d2m/actions/delete-message.js +++ b/d2m/actions/delete-message.js @@ -13,10 +13,11 @@ async function deleteMessage(data) { if (!roomID) return const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all() + db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(data.id) + db.prepare("DELETE FROM event_message WHERE message_id = ?").run(data.id) for (const eventID of eventsToRedact) { // Unfortunately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs await api.redactEvent(roomID, eventID) - db.prepare("DELETE FROM event_message WHERE event_id = ?").run(eventID) } } diff --git a/m2d/actions/redact.js b/m2d/actions/redact.js index 316b466..26a7142 100644 --- a/m2d/actions/redact.js +++ b/m2d/actions/redact.js @@ -13,7 +13,9 @@ const utils = sync.require("../converters/utils") */ async function deleteMessage(event) { const rows = from("event_message").join("message_channel", "message_id").select("channel_id", "message_id").where({event_id: event.redacts}).all() + db.prepare("DELETE FROM event_message WHERE event_id = ?").run(event.event_id) for (const row of rows) { + db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(row.message_id) discord.snow.channel.deleteMessage(row.channel_id, row.message_id, event.content.reason) } } diff --git a/m2d/actions/send-event.js b/m2d/actions/send-event.js index 1b60ffe..6b7d3b8 100644 --- a/m2d/actions/send-event.js +++ b/m2d/actions/send-event.js @@ -96,6 +96,8 @@ async function sendEvent(event) { } for (const id of messagesToDelete) { + db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(id) + db.prepare("DELETE FROM event_message WHERE message_id = ?").run(id) await channelWebhook.deleteMessageWithWebhook(channelID, id, threadID) } From d0a3c3ce2959766e991d6d9e219def20984beae2 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 10 Jan 2024 22:42:13 +1300 Subject: [PATCH 012/431] m->d: Remove rare "In reply to" fallback text --- m2d/converters/event-to-message.js | 2 +- m2d/converters/event-to-message.test.js | 96 +++++++++++++++++++++---- 2 files changed, 83 insertions(+), 15 deletions(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 37262ce..5cf8a04 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -429,7 +429,7 @@ async function eventToMessage(event, guild, di) { const repliedToContent = repliedToEvent.content.formatted_body || repliedToEvent.content.body const contentPreviewChunks = chunk( entities.decodeHTML5Strict( // Remove entities like & " - repliedToContent.replace(/.*<\/mx-reply>/, "") // Remove everything before replies, so just use the actual message body + repliedToContent.replace(/.*<\/mx-reply>/s, "") // Remove everything before replies, so just use the actual message body .replace(/^\s*
.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards .replace(/(?:\n|
)+/g, " ") // Should all be on one line .replace(/]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.) diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 8a57c41..d5f8455 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -713,6 +713,74 @@ test("event2message: rich reply to a sim user", async t => { ) }) +test("event2message: rich reply to a rich reply to a multi-line message should correctly strip reply fallback", async t => { + t.deepEqual( + await eventToMessage({ + content: { + body: "> <@cadence:cadence.moe> I just checked in a fix that will probably work, can you try reproducing this on the latest `main` branch and see if I fixed it?\n\nwill try later (tomorrow if I don't forgor)", + format: "org.matrix.custom.html", + formatted_body: "
In reply to @cadence:cadence.moe
I just checked in a fix that will probably work, can you try reproducing this on the latest main branch and see if I fixed it?
will try later (tomorrow if I don't forgor)", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$A0Rj559NKOh2VndCZSTJXcvgi42gZWVfVQt73wA2Hn0" + } + }, + msgtype: "m.text" + }, + origin_server_ts: 1704857452930, + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: {}, + event_id: "$Q5kNrPxGs31LfWOhUul5I03jNjlxKOwRmWVuivaqCHY", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!kLRqKKUQXcibIMtOpl:cadence.moe", "$A0Rj559NKOh2VndCZSTJXcvgi42gZWVfVQt73wA2Hn0", { + "type": "m.room.message", + "sender": "@cadence:cadence.moe", + "content": { + "msgtype": "m.text", + "body": "> <@solonovamax:matrix.org> multipart messages will be deleted if the message is edited to require less space\n> \n> \n> steps to reproduce:\n> \n> 1. send a message that is longer than 2000 characters (discord character limit)\n> - bot will split message into two messages on discord\n> 2. edit message to be under 2000 characters (discord character limit)\n> - bot will delete one of the messages on discord, and then edit the other one to include the edited content\n> - the bot will *then* delete the message on matrix (presumably) because one of the messages on discord was deleted (by \n\nI just checked in a fix that will probably work, can you try reproducing this on the latest `main` branch and see if I fixed it?", + "format": "org.matrix.custom.html", + "formatted_body": "
In reply to @solonovamax:matrix.org

multipart messages will be deleted if the message is edited to require less space

\n

steps to reproduce:

\n
    \n
  1. send a message that is longer than 2000 characters (discord character limit)
    2. \n
\n
    \n
  • bot will split message into two messages on discord
    • \n
\n
    \n
  1. edit message to be under 2000 characters (discord character limit)
    2. \n
\n
    \n
  • bot will delete one of the messages on discord, and then edit the other one to include the edited content
    • \n
  • the bot will then delete the message on matrix (presumably) because one of the messages on discord was deleted (by
    • \n
\n
I just checked in a fix that will probably work, can you try reproducing this on the latest main branch and see if I fixed it?", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$u4OD19vd2GETkOyhgFVla92oDKI4ojwBf2-JeVCG7EI" + } + } + }, + "origin_server_ts": 1704855484532, + "unsigned": { + "age": 19069564 + }, + "event_id": "$A0Rj559NKOh2VndCZSTJXcvgi42gZWVfVQt73wA2Hn0", + "room_id": "!cBxtVRxDlZvSVhJXVK:cadence.moe" + }) + }, + snow: { + guild: { + searchGuildMembers: (_, options) => { + t.fail(`should not search guild members, but actually searched for: ${options.query}`) + return [] + } + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "> <:L1:1144820033948762203><:L2:1144820084079087647>Ⓜ️**cadence [they]**:" + + "\n> I just checked in a fix that will probably work..." + + "\nwill try later (tomorrow if I don't forgor)", + avatar_url: undefined + }] + } + ) +}) + test("event2message: rich reply to an already-edited message will quote the new message content", async t => { t.deepEqual( await eventToMessage({ @@ -793,21 +861,21 @@ test("event2message: rich reply to an already-edited message will quote the new test("event2message: should avoid using blockquote contents as reply preview in rich reply to a sim user", async t => { t.deepEqual( await eventToMessage({ - type: "m.room.message", - sender: "@cadence:cadence.moe", - content: { - msgtype: "m.text", - body: "> <@_ooye_kyuugryphon:cadence.moe> > well, you said this, so...\n> \n> that can't be true! there's no way :o\n\nI agree!", - format: "org.matrix.custom.html", - formatted_body: "
In reply to @_ooye_kyuugryphon:cadence.moe
well, you said this, so...

that can't be true! there's no way :o
I agree!", - "m.relates_to": { - "m.in_reply_to": { - event_id: "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04" + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "> <@_ooye_kyuugryphon:cadence.moe> > well, you said this, so...\n> \n> that can't be true! there's no way :o\n\nI agree!", + format: "org.matrix.custom.html", + formatted_body: "
In reply to @_ooye_kyuugryphon:cadence.moe
well, you said this, so...

that can't be true! there's no way :o
I agree!", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04" + } } - } - }, - event_id: "$BpGx8_vqHyN6UQDARPDU51ftrlRBhleutRSgpAJJ--g", - room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" + }, + event_id: "$BpGx8_vqHyN6UQDARPDU51ftrlRBhleutRSgpAJJ--g", + room_id: "!fGgIymcYWOqjbSRUdV:cadence.moe" }, data.guild.general, { api: { getEvent: mockGetEvent(t, "!fGgIymcYWOqjbSRUdV:cadence.moe", "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04", { From a67708269daca916785e1682b7ca9972a92147ed Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 10 Jan 2024 22:46:20 +1300 Subject: [PATCH 013/431] m->d: Improve reply preview truncation punctuation --- m2d/converters/event-to-message.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 5cf8a04..31c0255 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -435,8 +435,8 @@ async function eventToMessage(event, guild, di) { .replace(/]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.) .replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting. ), 50) - contentPreview = ":\n> " - contentPreview += contentPreviewChunks.length > 1 ? contentPreviewChunks[0] + "..." : contentPreviewChunks[0] + contentPreview = ":\n> " + contentPreviewChunks[0] + if (contentPreviewChunks.length > 1) contentPreview = contentPreview.replace(/[,.']$/, "") + "..." } replyLine = `> ${replyLine}${contentPreview}\n` })() From 9efd6a49b85d32d200eb584736016475c0bcb84b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 10 Jan 2024 23:56:10 +1300 Subject: [PATCH 014/431] d->m: Sync missed emojis/stickers after restart --- d2m/actions/create-space.js | 28 ++++-- d2m/discord-packets.js | 1 + d2m/event-dispatcher.js | 12 ++- matrix/kstate.js | 9 +- package-lock.json | 171 +++++++++++++++++++++--------------- package.json | 1 + readme.md | 3 +- 7 files changed, 140 insertions(+), 85 deletions(-) diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js index b30f681..b86b3d6 100644 --- a/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -2,6 +2,7 @@ const assert = require("assert").strict const DiscordTypes = require("discord-api-types/v10") +const deepEqual = require("deep-equal") const reg = require("../../matrix/read-registration") const passthrough = require("../../passthrough") @@ -199,22 +200,33 @@ async function syncSpaceFully(guildID) { /** * @param {import("discord-api-types/v10").GatewayGuildEmojisUpdateDispatchData | import("discord-api-types/v10").GatewayGuildStickersUpdateDispatchData} data + * @param {boolean} checkBeforeSync false to always send new state, true to check the current state and only apply if state would change */ -async function syncSpaceExpressions(data) { +async function syncSpaceExpressions(data, checkBeforeSync) { // No need for kstate here. Each of these maps to a single state event, which will always overwrite what was there before. I can just send the state event. const spaceID = select("guild_space", "space_id", {guild_id: data.guild_id}).pluck().get() if (!spaceID) return - if ("emojis" in data && data.emojis.length) { - const content = await expression.emojisToState(data.emojis) - api.sendState(spaceID, "im.ponies.room_emotes", "moe.cadence.ooye.pack.emojis", content) + /** + * @typedef {import("discord-api-types/v10").GatewayGuildEmojisUpdateDispatchData & import("discord-api-types/v10").GatewayGuildStickersUpdateDispatchData} Expressions + * @param {string} spaceID + * @param {Expressions extends any ? keyof Expressions : never} key + * @param {string} eventKey + * @param {(emojis: any[]) => any} fn + */ + async function update(spaceID, key, eventKey, fn) { + if (!(key in data) || !data[key].length) return + const content = await fn(data[key]) + if (checkBeforeSync) { + const existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey) + if (deepEqual(existing, content, {strict: true})) return + } + api.sendState(spaceID, "im.ponies.room_emotes", eventKey, content) } - if ("stickers" in data && data.stickers.length) { - const content = await expression.stickersToState(data.stickers) - api.sendState(spaceID, "im.ponies.room_emotes", "moe.cadence.ooye.pack.stickers", content) - } + update(spaceID, "emojis", "moe.cadence.ooye.pack.emojis", expression.emojisToState) + update(spaceID, "stickers", "moe.cadence.ooye.pack.stickers", expression.stickersToState) } module.exports.createSpace = createSpace diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js index 4de84d9..b7093a8 100644 --- a/d2m/discord-packets.js +++ b/d2m/discord-packets.js @@ -42,6 +42,7 @@ const utils = { client.channels.set(thread.id, thread) } if (listen === "full") { + eventDispatcher.checkMissedExpressions(message.d) eventDispatcher.checkMissedMessages(client, message.d) } diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 12a3507..b586f61 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -125,6 +125,16 @@ module.exports = { } }, + /** + * When logging back in, check if we missed any changes to emojis or stickers. Apply the changes if so. + * @param {import("./discord-client")} client + * @param {DiscordTypes.GatewayGuildCreateDispatchData} guild + */ + async checkMissedExpressions(guild) { + const data = {guild_id: guild.id, ...guild} + createSpace.syncSpaceExpressions(data, true) + }, + /** * Announces to the parent room that the thread room has been created. * See notes.md, "Ignore MESSAGE_UPDATE and bridge THREAD_CREATE as the announcement" @@ -262,6 +272,6 @@ module.exports = { * @param {DiscordTypes.GatewayGuildEmojisUpdateDispatchData | DiscordTypes.GatewayGuildStickersUpdateDispatchData} data */ async onExpressionsUpdate(client, data) { - await createSpace.syncSpaceExpressions(data) + await createSpace.syncSpaceExpressions(data, false) } } diff --git a/matrix/kstate.js b/matrix/kstate.js index e37fb96..6358974 100644 --- a/matrix/kstate.js +++ b/matrix/kstate.js @@ -2,6 +2,7 @@ const assert = require("assert").strict const mixin = require("mixin-deep") +const deepEqual = require("deep-equal") /** Mutates the input. */ function kstateStripConditionals(kstate) { @@ -50,18 +51,14 @@ function diffKState(actual, target) { // Special handling for power levels, we want to deep merge the actual and target into the final state. if (!(key in actual)) throw new Error(`want to apply a power levels diff, but original power level data is missing\nstarted with: ${JSON.stringify(actual)}\nwant to apply: ${JSON.stringify(target)}`) const temp = mixin({}, actual[key], target[key]) - try { - assert.deepEqual(actual[key], temp) - } catch (e) { + if (!deepEqual(actual[key], temp, {strict: true})) { // they differ. use the newly prepared object as the diff. diff[key] = temp } } else if (key in actual) { // diff - try { - assert.deepEqual(actual[key], target[key]) - } catch (e) { + if (!deepEqual(actual[key], target[key], {strict: true})) { // they differ. use the target as the diff. diff[key] = target[key] } diff --git a/package-lock.json b/package-lock.json index 3367e93..054e701 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "better-sqlite3": "^9.0.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.9.5", + "deep-equal": "^2.2.3", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", "giframe": "github:cloudrac3r/giframe#v0.4.1", @@ -429,7 +430,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz", "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "is-array-buffer": "^3.0.1" @@ -462,7 +462,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -676,12 +675,13 @@ } }, "node_modules/call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", + "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.1", + "set-function-length": "^1.1.1" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -958,15 +958,14 @@ } }, "node_modules/deep-equal": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.1.tgz", - "integrity": "sha512-lKdkdV6EOGoVn65XaOsPdH4rMxTZOnmFyuIkMjM1i5HHCbfjC97dawgTAy0deYNfuqUqW+Q5VrVaQYtUpSd6yQ==", - "dev": true, + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.3.tgz", + "integrity": "sha512-ZIwpnevOurS8bpT4192sqAowWM76JDKSHYzMLty3BZGSswgq6pBaH3DhCSW5xVAZICZyKdOBPjwww5wfgT/6PA==", "dependencies": { "array-buffer-byte-length": "^1.0.0", - "call-bind": "^1.0.2", + "call-bind": "^1.0.5", "es-get-iterator": "^1.1.3", - "get-intrinsic": "^1.2.0", + "get-intrinsic": "^1.2.2", "is-arguments": "^1.1.1", "is-array-buffer": "^3.0.2", "is-date-object": "^1.0.5", @@ -976,11 +975,14 @@ "object-is": "^1.1.5", "object-keys": "^1.1.1", "object.assign": "^4.1.4", - "regexp.prototype.flags": "^1.5.0", + "regexp.prototype.flags": "^1.5.1", "side-channel": "^1.0.4", "which-boxed-primitive": "^1.0.2", "which-collection": "^1.0.1", - "which-typed-array": "^1.1.9" + "which-typed-array": "^1.1.13" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -994,11 +996,23 @@ "node": ">=4.0.0" } }, + "node_modules/define-data-property": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", + "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", + "dependencies": { + "get-intrinsic": "^1.2.1", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/define-properties": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.0.tgz", "integrity": "sha512-xvqAVKGfT1+UAvPwKTVw/njhdQ8ZhXK4lI0bCIuCMrp2up9nPnaDftrLtmpTazqd1o+UY4zgzU+avtMbDP+ldA==", - "dev": true, "dependencies": { "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" @@ -1113,7 +1127,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz", "integrity": "sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "get-intrinsic": "^1.1.3", @@ -1327,7 +1340,6 @@ "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dev": true, "dependencies": { "is-callable": "^1.1.3" } @@ -1396,15 +1408,17 @@ } }, "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/functions-have-names": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -1419,13 +1433,14 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", - "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", + "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -1474,7 +1489,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dev": true, "dependencies": { "get-intrinsic": "^1.1.3" }, @@ -1497,7 +1511,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -1515,7 +1528,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", - "dev": true, "dependencies": { "get-intrinsic": "^1.1.1" }, @@ -1523,6 +1535,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", @@ -1538,7 +1561,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "dev": true, "dependencies": { "has-symbols": "^1.0.2" }, @@ -1549,6 +1571,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hasown": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", + "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/heatsync": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/heatsync/-/heatsync-2.4.2.tgz", @@ -1645,7 +1678,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.5.tgz", "integrity": "sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ==", - "dev": true, "dependencies": { "get-intrinsic": "^1.2.0", "has": "^1.0.3", @@ -1667,7 +1699,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -1683,7 +1714,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz", "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "get-intrinsic": "^1.2.0", @@ -1702,7 +1732,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", - "dev": true, "dependencies": { "has-bigints": "^1.0.1" }, @@ -1714,7 +1743,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -1730,7 +1758,6 @@ "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -1754,7 +1781,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", - "dev": true, "dependencies": { "has-tostringtag": "^1.0.0" }, @@ -1778,7 +1804,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.2.tgz", "integrity": "sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -1787,7 +1812,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", - "dev": true, "dependencies": { "has-tostringtag": "^1.0.0" }, @@ -1802,7 +1826,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -1818,7 +1841,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.2.tgz", "integrity": "sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -1827,7 +1849,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", - "dev": true, "dependencies": { "call-bind": "^1.0.2" }, @@ -1839,7 +1860,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", - "dev": true, "dependencies": { "has-tostringtag": "^1.0.0" }, @@ -1854,7 +1874,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", - "dev": true, "dependencies": { "has-symbols": "^1.0.2" }, @@ -1869,7 +1888,6 @@ "version": "1.1.10", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.10.tgz", "integrity": "sha512-PJqgEHiWZvMpaFZ3uTc8kHPM4+4ADTlDniuQL7cU/UDA0Ql7F70yGfHph3cLNe+c9toaigv+DFzTJKhc2CtO6A==", - "dev": true, "dependencies": { "available-typed-arrays": "^1.0.5", "call-bind": "^1.0.2", @@ -1888,7 +1906,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz", "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -1897,7 +1914,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.2.tgz", "integrity": "sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "get-intrinsic": "^1.1.1" @@ -1909,8 +1925,7 @@ "node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", - "dev": true + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" }, "node_modules/isexe": { "version": "2.0.0", @@ -2236,7 +2251,6 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz", "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.3" @@ -2252,7 +2266,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, "engines": { "node": ">= 0.4" } @@ -2261,7 +2274,6 @@ "version": "4.1.4", "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz", "integrity": "sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==", - "dev": true, "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.1.4", @@ -2602,14 +2614,13 @@ } }, "node_modules/regexp.prototype.flags": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz", - "integrity": "sha512-0SutC3pNudRKgquxGoRGIz946MZVHqbNfPjBdxeOhBrdgDKlRoXmYLQN9xRbrR09ZXWeGAdPuif7egofn6v5LA==", - "dev": true, + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz", + "integrity": "sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg==", "dependencies": { "call-bind": "^1.0.2", "define-properties": "^1.2.0", - "functions-have-names": "^1.2.3" + "set-function-name": "^2.0.0" }, "engines": { "node": ">= 0.4" @@ -2789,6 +2800,33 @@ "node": ">= 0.8.0" } }, + "node_modules/set-function-length": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz", + "integrity": "sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==", + "dependencies": { + "define-data-property": "^1.1.1", + "get-intrinsic": "^1.2.1", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/set-function-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz", + "integrity": "sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==", + "dependencies": { + "define-data-property": "^1.0.1", + "functions-have-names": "^1.2.3", + "has-property-descriptors": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -2990,7 +3028,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz", "integrity": "sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==", - "dev": true, "dependencies": { "internal-slot": "^1.0.4" }, @@ -3492,7 +3529,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", - "dev": true, "dependencies": { "is-bigint": "^1.0.1", "is-boolean-object": "^1.1.0", @@ -3508,7 +3544,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.1.tgz", "integrity": "sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==", - "dev": true, "dependencies": { "is-map": "^2.0.1", "is-set": "^2.0.1", @@ -3520,17 +3555,15 @@ } }, "node_modules/which-typed-array": { - "version": "1.1.9", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.9.tgz", - "integrity": "sha512-w9c4xkx6mPidwp7180ckYWfMmvxpjlZuIudNtDf4N/tTAUB8VJbX25qZoAsrtGuYNnGw3pa0AXgbGKRB8/EceA==", - "dev": true, + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.13.tgz", + "integrity": "sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==", "dependencies": { "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", + "call-bind": "^1.0.4", "for-each": "^0.3.3", "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0", - "is-typed-array": "^1.1.10" + "has-tostringtag": "^1.0.0" }, "engines": { "node": ">= 0.4" diff --git a/package.json b/package.json index 01070d2..58a8674 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "better-sqlite3": "^9.0.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.9.5", + "deep-equal": "^2.2.3", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", "giframe": "github:cloudrac3r/giframe#v0.4.1", diff --git a/readme.md b/readme.md index 9b111ad..df09c7d 100644 --- a/readme.md +++ b/readme.md @@ -162,6 +162,7 @@ Follow these steps: * (1) chunk-text: It does what I want. * (0) cloudstorm: Discord gateway library with bring-your-own-caching that I trust. * (8) snowtransfer: Discord API library with bring-your-own-caching that I trust. +* (0) deep-equal: It's already pulled in by supertape. * (1) discord-markdown: This is my fork! * (0) giframe: This is my fork! * (1) heatsync: Module hot-reloader that I trust. @@ -169,7 +170,7 @@ Follow these steps: * (1) html-template-tag: This is my fork! * (1) js-yaml: It seems to do what I want, and it's already pulled in by matrix-appservice. * (70) matrix-appservice: I wish it didn't pull in express :( -* (0) minimist: It's already pulled in by better-sqlite3->prebuild-install +* (0) minimist: It's already pulled in by better-sqlite3->prebuild-install. * (0) mixin-deep: This is my fork! (It fixes a bug in regular mixin-deep.) * (3) node-fetch@2: I like it and it does what I want. * (0) pngjs: Lottie stickers are converted to bitmaps with the vendored Rlottie WASM build, then the bitmaps are converted to PNG with pngjs. From 20bab453ef7a2eb46087745a66b45f61794359ed Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 11 Jan 2024 15:56:42 +1300 Subject: [PATCH 015/431] d->m: Support bulk message deletion from bots --- d2m/actions/delete-message.js | 20 +++++++++++++++++++- d2m/discord-packets.js | 3 +++ d2m/event-dispatcher.js | 8 ++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/d2m/actions/delete-message.js b/d2m/actions/delete-message.js index f893e6d..cca5d25 100644 --- a/d2m/actions/delete-message.js +++ b/d2m/actions/delete-message.js @@ -1,7 +1,7 @@ // @ts-check const passthrough = require("../../passthrough") -const {sync, db, select} = passthrough +const {sync, db, select, from} = passthrough /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") @@ -21,4 +21,22 @@ async function deleteMessage(data) { } } +/** + * @param {import("discord-api-types/v10").GatewayMessageDeleteBulkDispatchData} data + */ +async function deleteMessageBulk(data) { + const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get() + if (!roomID) return + + const sids = JSON.stringify(data.ids) + const eventsToRedact = from("event_message").pluck("event_id").and("WHERE message_id IN (SELECT value FROM json_each(?)").all(sids) + db.prepare("DELETE FROM message_channel WHERE message_id IN (SELECT value FROM json_each(?)").run(sids) + db.prepare("DELETE FROM event_message WHERE message_id IN (SELECT value FROM json_each(?)").run(sids) + for (const eventID of eventsToRedact) { + // Awaiting will make it go slower, but since this could be a long-running operation either way, we want to leave rate limit capacity for other operations + await api.redactEvent(roomID, eventID) + } +} + module.exports.deleteMessage = deleteMessage +module.exports.deleteMessageBulk = deleteMessageBulk diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js index b7093a8..1129b71 100644 --- a/d2m/discord-packets.js +++ b/d2m/discord-packets.js @@ -153,6 +153,9 @@ const utils = { } else if (message.t === "MESSAGE_DELETE") { await eventDispatcher.onMessageDelete(client, message.d) + } else if (message.t === "MESSAGE_DELETE_BULK") { + await eventDispatcher.onMessageDeleteBulk(client, message.d) + } else if (message.t === "TYPING_START") { await eventDispatcher.onTypingStart(client, message.d) diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index b586f61..ecf960f 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -252,6 +252,14 @@ module.exports = { await deleteMessage.deleteMessage(data) }, + /** + * @param {import("./discord-client")} client + * @param {DiscordTypes.GatewayMessageDeleteBulkDispatchData} data + */ + async onMessageDeleteBulk(client, data) { + await deleteMessage.deleteMessageBulk(data) + }, + /** * @param {import("./discord-client")} client * @param {DiscordTypes.GatewayTypingStartDispatchData} data From 600ca1a11d4ec204b7daf8aaf28d0c08a1eb0f8f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 11 Jan 2024 15:56:58 +1300 Subject: [PATCH 016/431] Enable ts-check on event-dispatcher --- d2m/event-dispatcher.js | 48 ++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index ecf960f..7974be6 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -1,3 +1,5 @@ +// @ts-check + const assert = require("assert").strict const DiscordTypes = require("discord-api-types/v10") const util = require("util") @@ -47,27 +49,33 @@ module.exports = { if (Date.now() - lastReportedEvent < 5000) return lastReportedEvent = Date.now() - const channelID = gatewayMessage.d.channel_id + const channelID = gatewayMessage.d["channel_id"] if (!channelID) return const roomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() if (!roomID) return - let stackLines = e.stack.split("\n") - let cloudstormLine = stackLines.findIndex(l => l.includes("/node_modules/cloudstorm/")) - if (cloudstormLine !== -1) { - stackLines = stackLines.slice(0, cloudstormLine - 2) + let stackLines = null + if (e.stack) { + stackLines = e.stack.split("\n") + let cloudstormLine = stackLines.findIndex(l => l.includes("/node_modules/cloudstorm/")) + if (cloudstormLine !== -1) { + stackLines = stackLines.slice(0, cloudstormLine - 2) + } } + let formattedBody = "\u26a0 Bridged event from Discord not delivered" + + `
Gateway event: ${gatewayMessage.t}` + + `
${e.toString()}` + if (stackLines) { + formattedBody += `
Error trace` + + `
${stackLines.join("\n")}
` + } + formattedBody += `
Original payload` + + `
${util.inspect(gatewayMessage.d, false, 4, false)}
`, api.sendEvent(roomID, "m.room.message", { msgtype: "m.text", body: "\u26a0 Bridged event from Discord not delivered. See formatted content for full details.", format: "org.matrix.custom.html", - formatted_body: "\u26a0 Bridged event from Discord not delivered" - + `
Gateway event: ${gatewayMessage.t}` - + `
${e.toString()}` - + `
Error trace` - + `
${stackLines.join("\n")}
` - + `
Original payload` - + `
${util.inspect(gatewayMessage.d, false, 4, false)}
`, + formatted_body: formattedBody, "moe.cadence.ooye.error": { source: "discord", payload: gatewayMessage @@ -91,7 +99,7 @@ module.exports = { const prepared = select("event_message", "event_id", {}, "WHERE message_id = ?").pluck() for (const channel of guild.channels.concat(guild.threads)) { if (!bridgedChannels.includes(channel.id)) continue - if (!channel.last_message_id) continue + if (!("last_message_id" in channel) || !channel.last_message_id) continue const latestWasBridged = prepared.get(channel.last_message_id) if (latestWasBridged) continue @@ -116,7 +124,6 @@ module.exports = { for (let i = Math.min(messages.length, latestBridgedMessageIndex)-1; i >= 0; i--) { const simulatedGatewayDispatchData = { guild_id: guild.id, - mentions: [], backfill: true, ...messages[i] } @@ -127,7 +134,6 @@ module.exports = { /** * When logging back in, check if we missed any changes to emojis or stickers. Apply the changes if so. - * @param {import("./discord-client")} client * @param {DiscordTypes.GatewayGuildCreateDispatchData} guild */ async checkMissedExpressions(guild) { @@ -142,7 +148,8 @@ module.exports = { * @param {DiscordTypes.APIThreadChannel} thread */ async onThreadCreate(client, thread) { - const parentRoomID = select("channel_room", "room_id", {channel_id: thread.parent_id}).pluck().get() + const channelID = thread.parent_id || undefined + const parentRoomID = select("channel_room", "room_id", {channel_id: channelID}).pluck().get() if (!parentRoomID) return // Not interested in a thread if we aren't interested in its wider channel const threadRoomID = await createRoom.syncRoom(thread.id) // Create room (will share the same inflight as the initial message to the thread) await announceThread.announceThread(parentRoomID, threadRoomID, thread) @@ -192,10 +199,10 @@ module.exports = { return } } - /** @type {DiscordTypes.APIGuildChannel} */ const channel = client.channels.get(message.channel_id) - if (!channel.guild_id) return // Nothing we can do in direct messages. + if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages. const guild = client.guilds.get(channel.guild_id) + assert(guild) await sendMessage.sendMessage(message, guild), await discordCommandHandler.execute(message, channel, guild) @@ -217,11 +224,12 @@ module.exports = { // If the message content is a string then it includes all interesting fields and is meaningful. if (typeof data.content === "string") { /** @type {DiscordTypes.GatewayMessageCreateDispatchData} */ + // @ts-ignore const message = data - /** @type {DiscordTypes.APIGuildChannel} */ const channel = client.channels.get(message.channel_id) - if (!channel.guild_id) return // Nothing we can do in direct messages. + if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages. const guild = client.guilds.get(channel.guild_id) + assert(guild) await editMessage.editMessage(message, guild) } }, From 7d4379a099b84a4bc1a76de2a5d8ef54ddc1050d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 12 Jan 2024 14:33:23 +1300 Subject: [PATCH 017/431] Fix error in startup emojis check --- d2m/actions/create-space.js | 8 +++++++- matrix/mreq.js | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js index b86b3d6..0439496 100644 --- a/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -219,7 +219,13 @@ async function syncSpaceExpressions(data, checkBeforeSync) { if (!(key in data) || !data[key].length) return const content = await fn(data[key]) if (checkBeforeSync) { - const existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey) + let existing + try { + existing = await api.getStateEvent(spaceID, "im.ponies.room_emotes", eventKey) + } catch (e) { + // State event not found. This space doesn't have any existing emojis. We create a dummy empty event for comparison's sake. + existing = fn([]) + } if (deepEqual(existing, content, {strict: true})) return } api.sendState(spaceID, "im.ponies.room_emotes", eventKey, content) diff --git a/matrix/mreq.js b/matrix/mreq.js index 4291ede..0bd7505 100644 --- a/matrix/mreq.js +++ b/matrix/mreq.js @@ -39,7 +39,7 @@ async function mreq(method, url, body, extra = {}) { const res = await fetch(baseUrl + url, opts) const root = await res.json() - if (!res.ok || root.errcode) throw new MatrixServerError(root, opts) + if (!res.ok || root.errcode) throw new MatrixServerError(root, {baseUrl, url, ...opts}) return root } From b060451bafd620a581b6135a10af5167610a7e4e Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 16 Jan 2024 16:00:33 +1300 Subject: [PATCH 018/431] Minor documentation rewording --- d2m/actions/create-space.js | 8 ++++---- d2m/converters/message-to-event.js | 2 +- readme.md | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js index 0439496..4e30bcd 100644 --- a/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -22,7 +22,7 @@ const ks = sync.require("../../matrix/kstate") const inflightSpaceCreate = new Map() /** - * @param {import("discord-api-types/v10").RESTGetAPIGuildResult} guild + * @param {DiscordTypes.RESTGetAPIGuildResult} guild * @param {any} kstate */ async function createSpace(guild, kstate) { @@ -199,7 +199,7 @@ async function syncSpaceFully(guildID) { } /** - * @param {import("discord-api-types/v10").GatewayGuildEmojisUpdateDispatchData | import("discord-api-types/v10").GatewayGuildStickersUpdateDispatchData} data + * @param {DiscordTypes.GatewayGuildEmojisUpdateDispatchData | DiscordTypes.GatewayGuildStickersUpdateDispatchData} data * @param {boolean} checkBeforeSync false to always send new state, true to check the current state and only apply if state would change */ async function syncSpaceExpressions(data, checkBeforeSync) { @@ -209,11 +209,11 @@ async function syncSpaceExpressions(data, checkBeforeSync) { if (!spaceID) return /** - * @typedef {import("discord-api-types/v10").GatewayGuildEmojisUpdateDispatchData & import("discord-api-types/v10").GatewayGuildStickersUpdateDispatchData} Expressions + * @typedef {DiscordTypes.GatewayGuildEmojisUpdateDispatchData & DiscordTypes.GatewayGuildStickersUpdateDispatchData} Expressions * @param {string} spaceID * @param {Expressions extends any ? keyof Expressions : never} key * @param {string} eventKey - * @param {(emojis: any[]) => any} fn + * @param {typeof expression["emojisToState"] | typeof expression["stickersToState"]} fn */ async function update(spaceID, key, eventKey, fn) { if (!(key in data) || !data[key].length) return diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 60aba95..cb9e900 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -437,7 +437,7 @@ async function messageToEvent(message, guild, options = {}, di) { // Then embeds for (const embed of message.embeds || []) { if (embed.type === "image") { - continue // Matrix's own image embeds are fine. + continue // Matrix's own URL previews are fine for images. } // Start building up a replica ("rep") of the embed in Discord-markdown format, which we will convert into both plaintext and formatted body at once diff --git a/readme.md b/readme.md index df09c7d..e7baf18 100644 --- a/readme.md +++ b/readme.md @@ -53,7 +53,7 @@ For more information about features, [see the user guide.](https://gitdab.com/ca ## Efficiency details -Using WeatherStack as a thin layer between the bridge application and the Discord API lets us control exactly what data is cached. Only necessary information is cached. For example, member data, user data, message content, and past edits are never stored in memory. This keeps the memory usage low and also prevents it ballooning in size over the bridge's runtime. +Using WeatherStack as a thin layer between the bridge application and the Discord API lets us control exactly what data is cached in memory. Only necessary information is cached. For example, member data, user data, message content, and past edits are never stored in memory. This keeps the memory usage low and also prevents it ballooning in size over the bridge's runtime. The bridge uses a small SQLite database to store relationships like which Discord messages correspond to which Matrix messages. This is so the bridge knows what to edit when some message is edited on Discord. Using `without rowid` on the database tables stores the index and the data in the same B-tree. Since Matrix and Discord's internal IDs are quite long, this vastly reduces storage space because those IDs do not have to be stored twice separately. Some event IDs are actually stored as xxhash integers to reduce storage requirements even more. On my personal instance of OOYE, every 100,000 messages require 16.1 MB of storage space in the SQLite database. @@ -61,11 +61,11 @@ Only necessary data and columns are queried from the database. We only contact t File uploads (like avatars from bridged members) are checked locally and deduplicated. Only brand new files are uploaded to the homeserver. This saves loads of space in the homeserver's media repo, especially for Synapse. -Switching to [WAL mode](https://www.sqlite.org/wal.html) could improve your database access speed even more. Run `node scripts/wal.js` if you want to switch to WAL mode. +Switching to [WAL mode](https://www.sqlite.org/wal.html) could improve your database access speed even more. Run `node scripts/wal.js` if you want to switch to WAL mode. This will also enable `synchronous = NORMAL`. # Setup -If you get stuck, you're welcome to message @cadence:cadence.moe to ask for help setting up OOYE! +If you get stuck, you're welcome to message [#out-of-your-element:cadence.moe](https://matrix.to/#/#out-of-your-element:cadence.moe) or [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe) to ask for help setting up OOYE! You'll need: From ed7404ea19030706fbf728d4a280b36353b8a84d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 16 Jan 2024 16:02:31 +1300 Subject: [PATCH 019/431] Fix #16 pinned messages order --- d2m/converters/pins-to-list.js | 1 + d2m/converters/pins-to-list.test.js | 6 +-- docs/how-to-add-a-new-event-type.md | 82 ++++++++++++++++++++++++++--- 3 files changed, 79 insertions(+), 10 deletions(-) diff --git a/d2m/converters/pins-to-list.js b/d2m/converters/pins-to-list.js index f401de2..047bb9f 100644 --- a/d2m/converters/pins-to-list.js +++ b/d2m/converters/pins-to-list.js @@ -12,6 +12,7 @@ function pinsToList(pins) { const eventID = select("event_message", "event_id", {message_id: message.id, part: 0}).pluck().get() if (eventID) result.push(eventID) } + result.reverse() return result } diff --git a/d2m/converters/pins-to-list.test.js b/d2m/converters/pins-to-list.test.js index c2e3774..92e5678 100644 --- a/d2m/converters/pins-to-list.test.js +++ b/d2m/converters/pins-to-list.test.js @@ -5,8 +5,8 @@ const {pinsToList} = require("./pins-to-list") test("pins2list: converts known IDs, ignores unknown IDs", t => { const result = pinsToList(data.pins.faked) t.deepEqual(result, [ - "$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg", - "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA", - "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4" + "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4", + "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuAno", + "$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg" ]) }) diff --git a/docs/how-to-add-a-new-event-type.md b/docs/how-to-add-a-new-event-type.md index 10beec1..ea808e0 100644 --- a/docs/how-to-add-a-new-event-type.md +++ b/docs/how-to-add-a-new-event-type.md @@ -201,15 +201,15 @@ Good to go. at module.exports (out-of-your-element/node_modules/try-to-catch/lib/try-to-catch.js:7:29) ``` -Oh, this was actually an accident, I didn't make it fail for demonstration purposes! Let's see what this bug is. It's returning the right number of IDs, but 2 out of the 3 are incorrect. The green `-` lines are "expected" and the red `+` lines are "actual". I should check where that wrong ID `$51f...` got taken from. +Oh no! (I promise I didn't make it fail for demonstration purposes, this was actually an accident!) Let's see what this bug is. It's returning the right number of IDs, but 2 out of the 3 are incorrect. The green `-` lines are "expected" and the red `+` lines are "actual". The wrong ID `$51f...` must have been taken from _somewhere_ in the test data, so I'll first search the codebase and find where it came from: -``` - - snip - ooye-test-data.sql +```sql +-- snipped from ooye-test-data.sql ('$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA', 'm.room.message', 'm.text', '1141501302736695316', 0, 1), ('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', 1, 1), ``` -Context: This Discord message `1141501302736695316` is actually part of 2 different Matrix events, `$mtR...` and `$51f...`. This often happens when a Discord user uploads an image with a caption. Matrix doesn't support combined image+text events, so the image and the text have to be bridged to separate events. We should consider the text to be the primary part, and pin that, and consider the image to be the secondary part, and not pin that. +Explanation: This Discord message `1141501302736695316` is actually part of 2 different Matrix events, `$mtR...` and `$51f...`. This often happens when a Discord user uploads an image with a caption. Matrix doesn't support combined image+text events, so the image and the text have to be bridged to separate events. We should consider the text to be the primary part, and pin that, and consider the image to be the secondary part, and not pin that. We already have a column `part` in the `event_message` table for this reason! When `part = 0`, that's the primary part. I'll edit the converter to actually use that column: @@ -269,7 +269,7 @@ index 83c31cd..4de84d9 100644 await eventDispatcher.onThreadCreate(client, message.d) ``` -`event-dispatcher.js` will now check if the event seems reasonable and is allowed in this context. For example, we can only update pins if the channel is actually bridged somewhere. This should be another quick check which passes to an action to do the API calls: +`event-dispatcher.js` will now check if the event seems reasonable and is allowed in this context. For example, we can only update pins if the channel is actually bridged somewhere. After the check, we'll call the action: ```diff diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js @@ -304,7 +304,7 @@ index 0f9f1e6..6e91e9e 100644 * @param {DiscordTypes.GatewayMessageCreateDispatchData} message ``` -And now I can create the `update-pins.js` action: +And now I can write the `update-pins.js` action: ```diff diff --git a/d2m/actions/update-pins.js b/d2m/actions/update-pins.js @@ -339,6 +339,74 @@ index 0000000..40cc358 I try to keep as much logic as possible out of the actions and in the converters. This should mean I *never have to unit test the actions themselves.* The actions will be tested manually with the real bot. +## See if it works + +``` +node start.js +``` + +We can try these things and see if they are bridged to Matrix: + +- Pin a recent message on Discord-side +- Pin an old message on Discord-side +- Unpin a message on Discord-side + +It works like I'd expect! + +## Order of pinned messages + +I expected that to be the end of the guide, but after some time, I noticed a new problem: The pins are in reverse order. How could this happen? + +[After some investigation,](https://gitdab.com/cadence/out-of-your-element/issues/16) it turns out Discord puts the most recently pinned message at the start of the array and displays the array in forwards order, while Matrix puts the most recently pinned message at the end of the array and displays the array in reverse order. + +We'll fix this by reversing the order of the list of pins before we store it. I'll do this in the converter. + +```diff +diff --git a/d2m/converters/pins-to-list.js b/d2m/converters/pins-to-list.js +index f401de2..047bb9f 100644 +--- a/d2m/converters/pins-to-list.js ++++ b/d2m/converters/pins-to-list.js +@@ -12,6 +12,7 @@ function pinsToList(pins) { + const eventID = select("event_message", "event_id", {message_id: message.id, part: 0}).pluck().get() + if (eventID) result.push(eventID) + } ++ result.reverse() + return result + } +``` + +Since the results have changed, I'll need to update the test so it expects the new result: + +```diff +diff --git a/d2m/converters/pins-to-list.test.js b/d2m/converters/pins-to-list.test.js +index c2e3774..92e5678 100644 +--- a/d2m/converters/pins-to-list.test.js ++++ b/d2m/converters/pins-to-list.test.js +@@ -5,8 +5,8 @@ const {pinsToList} = require("./pins-to-list") + test("pins2list: converts known IDs, ignores unknown IDs", t => { + const result = pinsToList(data.pins.faked) + t.deepEqual(result, [ +- "$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg", +- "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA", +- "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4" ++ "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4", ++ "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA", ++ "$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg" + ]) + }) +``` + +``` +><> $ npm t + + 144 tests + 232 passed + + Pass! +``` + +Next time a message is pinned or unpinned on Discord, the order should be updated on Matrix. + ## Notes on missed events Note that this will only sync pins _when the pins change._ Existing pins from Discord will not be backfilled to Matrix rooms. If I wanted, there's a couple of ways I could address this: @@ -346,4 +414,4 @@ Note that this will only sync pins _when the pins change._ Existing pins from Di * I could create a one-shot script in `scripts/update-pins.js` which will sync pins for _all_ Discord channels right away. I can run this after finishing the feature, or if the bot has been offline for some time. * I could create a database table that holds the timestamp of the most recently detected pin for each channel - the `last_pin_timestamp` field from the gateway. Every time the bot starts, it would automatically compare the database table against every channel, and if the pins have changed since it last looked, it could automatically update them. -I already have a mechanism for backfilling missed messages when the bridge starts up. Option 2 there would add a similar feature for backfilling missed pins. That could be worth considering, but it's less important and more complex. Perhaps we'll come back to it. +I already have code to backfill missed messages when the bridge starts up. The second option above would add a similar feature for backfilling missed pins. It would be worth considering. From 8987107685f8748288be0e247163c5722195d2c3 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 18 Jan 2024 00:30:55 +1300 Subject: [PATCH 020/431] Backfill missed pins and pins from the past --- d2m/actions/update-pins.js | 33 +++++++--- d2m/converters/pins-to-list.test.js | 2 +- d2m/discord-packets.js | 8 +++ d2m/event-dispatcher.js | 45 ++++++++++++- .../0008-add-last-bridged-pin-timestamp.sql | 5 ++ db/orm-defs.d.ts | 1 + discord/utils.js | 2 +- discord/utils.test.js | 63 ++++++++++++++++++- stdin.js | 1 + 9 files changed, 147 insertions(+), 13 deletions(-) create mode 100644 db/migrations/0008-add-last-bridged-pin-timestamp.sql diff --git a/d2m/actions/update-pins.js b/d2m/actions/update-pins.js index 40cc358..5d98501 100644 --- a/d2m/actions/update-pins.js +++ b/d2m/actions/update-pins.js @@ -1,22 +1,37 @@ // @ts-check const passthrough = require("../../passthrough") -const {discord, sync} = passthrough +const {discord, sync, db} = passthrough /** @type {import("../converters/pins-to-list")} */ const pinsToList = sync.require("../converters/pins-to-list") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") /** - * @param {string} channelID - * @param {string} roomID + * @template {string | null | undefined} T + * @param {T} timestamp + * @returns {T extends string ? number : null} */ -async function updatePins(channelID, roomID) { - const pins = await discord.snow.channel.getChannelPinnedMessages(channelID) - const eventIDs = pinsToList.pinsToList(pins) - await api.sendState(roomID, "m.room.pinned_events", "", { - pinned: eventIDs - }) +function convertTimestamp(timestamp) { + // @ts-ignore + return typeof timestamp === "string" ? Math.floor(new Date(timestamp).getTime() / 1000) : null } +/** + * @param {string} channelID + * @param {string} roomID + * @param {number?} convertedTimestamp + */ +async function updatePins(channelID, roomID, convertedTimestamp) { + const pins = await discord.snow.channel.getChannelPinnedMessages(channelID) + const eventIDs = pinsToList.pinsToList(pins) + if (pins.length === eventIDs.length || eventIDs.length) { + await api.sendState(roomID, "m.room.pinned_events", "", { + pinned: eventIDs + }) + } + db.prepare("UPDATE channel_room SET last_bridged_pin_timestamp = ? WHERE channel_id = ?").run(convertedTimestamp || 0, channelID) +} + +module.exports.convertTimestamp = convertTimestamp module.exports.updatePins = updatePins diff --git a/d2m/converters/pins-to-list.test.js b/d2m/converters/pins-to-list.test.js index 92e5678..8a6daea 100644 --- a/d2m/converters/pins-to-list.test.js +++ b/d2m/converters/pins-to-list.test.js @@ -6,7 +6,7 @@ test("pins2list: converts known IDs, ignores unknown IDs", t => { const result = pinsToList(data.pins.faked) t.deepEqual(result, [ "$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4", - "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuAno", + "$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA", "$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg" ]) }) diff --git a/d2m/discord-packets.js b/d2m/discord-packets.js index 1129b71..6ba839a 100644 --- a/d2m/discord-packets.js +++ b/d2m/discord-packets.js @@ -43,6 +43,7 @@ const utils = { } if (listen === "full") { eventDispatcher.checkMissedExpressions(message.d) + eventDispatcher.checkMissedPins(client, message.d) eventDispatcher.checkMissedMessages(client, message.d) } @@ -94,6 +95,13 @@ const utils = { client.channels.set(message.d.id, message.d) + } else if (message.t === "CHANNEL_PINS_UPDATE") { + const channel = client.channels.get(message.d.channel_id) + if (channel) { + channel["last_pin_timestamp"] = message.d.last_pin_timestamp + } + + } else if (message.t === "GUILD_DELETE") { client.guilds.delete(message.d.id) const channels = client.guildChannelMap.get(message.d.id) diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 7974be6..6c872fb 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -25,9 +25,15 @@ const createSpace = sync.require("./actions/create-space") const updatePins = sync.require("./actions/update-pins") /** @type {import("../matrix/api")}) */ const api = sync.require("../matrix/api") +/** @type {import("../discord/utils")} */ +const utils = sync.require("../discord/utils") /** @type {import("../discord/discord-command-handler")}) */ const discordCommandHandler = sync.require("../discord/discord-command-handler") +/** @type {any} */ // @ts-ignore bad types from semaphore +const Semaphore = require("@chriscdn/promise-semaphore") +const checkMissedPinsSema = new Semaphore() + let lastReportedEvent = 0 // Grab Discord events we care about for the bridge, check them, and pass them on @@ -103,6 +109,14 @@ module.exports = { const latestWasBridged = prepared.get(channel.last_message_id) if (latestWasBridged) continue + // Permissions check + const member = guild.members.find(m => m.user?.id === client.user.id) + if (!member) return + if (!("permission_overwrites" in channel)) continue + const permissions = utils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites) + const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY + if ((permissions & wants) !== wants) continue // We don't have permission to look back in this channel + /** More recent messages come first. */ // console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`) let messages @@ -132,6 +146,34 @@ module.exports = { } }, + /** + * When logging back in, check if the pins on Matrix-side are up to date. If they aren't, update all pins. + * Rather than query every room on Matrix-side, we cache the latest pinned message in the database and compare against that. + * @param {import("./discord-client")} client + * @param {DiscordTypes.GatewayGuildCreateDispatchData} guild + */ + async checkMissedPins(client, guild) { + if (guild.unavailable) return + const member = guild.members.find(m => m.user?.id === client.user.id) + if (!member) return + for (const channel of guild.channels) { + if (!("last_pin_timestamp" in channel) || !channel.last_pin_timestamp) continue // Only care about channels that have pins + if (!("permission_overwrites" in channel)) continue + const lastPin = updatePins.convertTimestamp(channel.last_pin_timestamp) + + // Permissions check + const permissions = utils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites) + const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY + if ((permissions & wants) !== wants) continue // We don't have permission to look up the pins in this channel + + const row = select("channel_room", ["room_id", "last_bridged_pin_timestamp"], {channel_id: channel.id}).get() + if (!row) continue // Only care about already bridged channels + if (row.last_bridged_pin_timestamp == null || lastPin > row.last_bridged_pin_timestamp) { + checkMissedPinsSema.request(() => updatePins.updatePins(channel.id, row.room_id, lastPin)) + } + } + }, + /** * When logging back in, check if we missed any changes to emojis or stickers. Apply the changes if so. * @param {DiscordTypes.GatewayGuildCreateDispatchData} guild @@ -183,7 +225,8 @@ module.exports = { async onChannelPinsUpdate(client, data) { const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get() if (!roomID) return // No target room to update pins in - await updatePins.updatePins(data.channel_id, roomID) + const convertedTimestamp = updatePins.convertTimestamp(data.last_pin_timestamp) + await updatePins.updatePins(data.channel_id, roomID, convertedTimestamp) }, /** diff --git a/db/migrations/0008-add-last-bridged-pin-timestamp.sql b/db/migrations/0008-add-last-bridged-pin-timestamp.sql new file mode 100644 index 0000000..dbcb81d --- /dev/null +++ b/db/migrations/0008-add-last-bridged-pin-timestamp.sql @@ -0,0 +1,5 @@ +BEGIN TRANSACTION; + +ALTER TABLE channel_room ADD COLUMN last_bridged_pin_timestamp INTEGER; + +COMMIT; diff --git a/db/orm-defs.d.ts b/db/orm-defs.d.ts index ec8d498..64e5c77 100644 --- a/db/orm-defs.d.ts +++ b/db/orm-defs.d.ts @@ -6,6 +6,7 @@ export type Models = { nick: string | null thread_parent: string | null custom_avatar: string | null + last_bridged_pin_timestamp: number | null } event_message: { diff --git a/discord/utils.js b/discord/utils.js index ceb4468..6788bcf 100644 --- a/discord/utils.js +++ b/discord/utils.js @@ -34,7 +34,7 @@ function getPermissions(userRoles, guildRoles, userID, channelOverwrites) { // Role deny overwrite => userRoles.includes(overwrite.id) && (allowed &= ~BigInt(overwrite.deny)), // Role allow - overwrite => userRoles.includes(overwrite.id) && (allowed |= ~BigInt(overwrite.allow)), + overwrite => userRoles.includes(overwrite.id) && (allowed |= BigInt(overwrite.allow)), // User deny overwrite => overwrite.id === userID && (allowed &= ~BigInt(overwrite.deny)), // User allow diff --git a/discord/utils.test.js b/discord/utils.test.js index fd064ef..1f3783f 100644 --- a/discord/utils.test.js +++ b/discord/utils.test.js @@ -18,6 +18,67 @@ test("discord utils: converts snowflake to timestamp", t => { t.equal(utils.snowflakeToTimestampExact("86913608335773696"), 1440792219004) }) -test("discerd utils: converts timestamp to snowflake", t => { +test("discord utils: converts timestamp to snowflake", t => { t.match(utils.timestampToSnowflakeInexact(1440792219004), /^869136083357.....$/) }) + +test("getPermissions: channel overwrite to allow role works", t => { + const guildRoles = [ + { + version: 1695412489043, + unicode_emoji: null, + tags: {}, + position: 0, + permissions: "559623605571137", + name: "@everyone", + mentionable: false, + managed: false, + id: "1154868424724463687", + icon: null, + hoist: false, + flags: 0, + color: 0 + }, + { + version: 1695412604262, + unicode_emoji: null, + tags: { bot_id: "466378653216014359" }, + position: 1, + permissions: "536995904", + name: "PluralKit", + mentionable: false, + managed: true, + id: "1154868908336099444", + icon: null, + hoist: false, + flags: 0, + color: 0 + }, + { + version: 1698778936921, + unicode_emoji: null, + tags: {}, + position: 1, + permissions: "536870912", + name: "web hookers", + mentionable: false, + managed: false, + id: "1168988246680801360", + icon: null, + hoist: false, + flags: 0, + color: 0 + } + ] + const userRoles = [ "1168988246680801360" ] + const userID = "684280192553844747" + const overwrites = [ + { type: 0, id: "1154868908336099444", deny: "0", allow: "1024" }, + { type: 0, id: "1154868424724463687", deny: "1024", allow: "0" }, + { type: 0, id: "1168988246680801360", deny: "0", allow: "1024" }, + { type: 1, id: "353373325575323648", deny: "0", allow: "1024" } + ] + const permissions = utils.getPermissions(userRoles, guildRoles, userID, overwrites) + const want = BigInt(1 << 10 | 1 << 16) + t.equal((permissions & want), want) +}) diff --git a/stdin.js b/stdin.js index 587b176..da69d7c 100644 --- a/stdin.js +++ b/stdin.js @@ -16,6 +16,7 @@ const api = sync.require("./matrix/api") const file = sync.require("./matrix/file") const sendEvent = sync.require("./m2d/actions/send-event") const eventDispatcher = sync.require("./d2m/event-dispatcher") +const updatePins = sync.require("./d2m/actions/update-pins") const ks = sync.require("./matrix/kstate") const guildID = "112760669178241024" From 64beb6c996084e5500c6e160e675259ca932323f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 18 Jan 2024 16:54:09 +1300 Subject: [PATCH 021/431] Code coverage for lottie --- d2m/actions/lottie.js | 53 +++++++++++++++++++++++++++++ d2m/converters/lottie.js | 45 ++++++------------------ d2m/converters/lottie.test.js | 34 ++++++++++++++++++ d2m/converters/message-to-event.js | 4 +-- test/res/lottie-bee.json | 1 + test/res/lottie-bee.license.txt | 16 +++++++++ test/res/lottie-bee.png | Bin 0 -> 5492 bytes test/test.js | 1 + 8 files changed, 117 insertions(+), 37 deletions(-) create mode 100644 d2m/actions/lottie.js create mode 100644 d2m/converters/lottie.test.js create mode 100644 test/res/lottie-bee.json create mode 100644 test/res/lottie-bee.license.txt create mode 100644 test/res/lottie-bee.png diff --git a/d2m/actions/lottie.js b/d2m/actions/lottie.js new file mode 100644 index 0000000..4635fed --- /dev/null +++ b/d2m/actions/lottie.js @@ -0,0 +1,53 @@ +// @ts-check + +const DiscordTypes = require("discord-api-types/v10") +const Ty = require("../../types") +const assert = require("assert").strict + +const passthrough = require("../../passthrough") +const {sync, db, select} = passthrough +/** @type {import("../../matrix/file")} */ +const file = sync.require("../../matrix/file") +/** @type {import("../../matrix/mreq")} */ +const mreq = sync.require("../../matrix/mreq") +/** @type {import("../converters/lottie")} */ +const convertLottie = sync.require("../converters/lottie") + +const INFO = { + mimetype: "image/png", + w: convertLottie.SIZE, + h: convertLottie.SIZE +} + +/** + * @param {DiscordTypes.APIStickerItem} stickerItem + * @returns {Promise<{mxc_url: string, info: typeof INFO}>} + */ +async function convert(stickerItem) { + // Reuse sticker if already converted and uploaded + const existingMxc = select("lottie", "mxc_url", {sticker_id: stickerItem.id}).pluck().get() + if (existingMxc) return {mxc_url: existingMxc, info: INFO} + + // Fetch sticker data from Discord + const res = await fetch(file.DISCORD_IMAGES_BASE + file.sticker(stickerItem)) + if (res.status !== 200) throw new Error("Sticker data file not found.") + const text = await res.text() + + // Convert to PNG (readable stream) + const readablePng = await convertLottie.convert(text) + + // Upload to MXC + /** @type {Ty.R.FileUploaded} */ + const root = await mreq.mreq("POST", "/media/v3/upload", readablePng, { + headers: { + "Content-Type": INFO.mimetype + } + }) + assert(root.content_uri) + + // Save the link for next time + db.prepare("INSERT INTO lottie (sticker_id, mxc_url) VALUES (?, ?)").run(stickerItem.id, root.content_uri) + return {mxc_url: root.content_uri, info: INFO} +} + +module.exports.convert = convert diff --git a/d2m/converters/lottie.js b/d2m/converters/lottie.js index a0d1cd1..f802e35 100644 --- a/d2m/converters/lottie.js +++ b/d2m/converters/lottie.js @@ -1,25 +1,10 @@ // @ts-check -const DiscordTypes = require("discord-api-types/v10") -const Ty = require("../../types") -const assert = require("assert").strict +const stream = require("stream") const {PNG} = require("pngjs") -const passthrough = require("../../passthrough") -const {sync, db, discord, select} = passthrough -/** @type {import("../../matrix/file")} */ -const file = sync.require("../../matrix/file") -//** @type {import("../../matrix/mreq")} */ -const mreq = sync.require("../../matrix/mreq") - const SIZE = 160 // Discord's display size on 1x displays is 160 -const INFO = { - mimetype: "image/png", - w: SIZE, - h: SIZE -} - /** * @typedef RlottieWasm * @prop {(string) => boolean} load load lottie data from string of json @@ -34,16 +19,11 @@ const Rlottie = (async () => { })() /** - * @param {DiscordTypes.APIStickerItem} stickerItem - * @returns {Promise<{mxc_url: string, info: typeof INFO}>} + * @param {string} text + * @returns {Promise} */ -async function convert(stickerItem) { - const existingMxc = select("lottie", "mxc_url", {sticker_id: stickerItem.id}).pluck().get() - if (existingMxc) return {mxc_url: existingMxc, info: INFO} +async function convert(text) { const r = await Rlottie - const res = await fetch(file.DISCORD_IMAGES_BASE + file.sticker(stickerItem)) - if (res.status !== 200) throw new Error("Sticker data file not found.") - const text = await res.text() /** @type RlottieWasm */ const rh = new r.RlottieWasm() const status = rh.load(text) @@ -58,17 +38,12 @@ async function convert(stickerItem) { inputHasAlpha: true, }) png.data = Buffer.from(rendered) - // @ts-ignore wrong type from pngjs - const readablePng = png.pack() - /** @type {Ty.R.FileUploaded} */ - const root = await mreq.mreq("POST", "/media/v3/upload", readablePng, { - headers: { - "Content-Type": INFO.mimetype - } - }) - assert(root.content_uri) - db.prepare("INSERT INTO lottie (sticker_id, mxc_url) VALUES (?, ?)").run(stickerItem.id, root.content_uri) - return {mxc_url: root.content_uri, info: INFO} + // The transform stream is necessary because PNG requires me to pipe it somewhere before this event loop ends + const resultStream = png.pack() + const p = new stream.PassThrough() + resultStream.pipe(p) + return p } module.exports.convert = convert +module.exports.SIZE = SIZE diff --git a/d2m/converters/lottie.test.js b/d2m/converters/lottie.test.js new file mode 100644 index 0000000..9d9255b --- /dev/null +++ b/d2m/converters/lottie.test.js @@ -0,0 +1,34 @@ +// @ts-check + +const fs = require("fs") +const stream = require("stream") +const {test} = require("supertape") +const {convert} = require("./lottie") + +const WRITE_PNG = false + +test("lottie: can convert and save PNG", async t => { + const input = await fs.promises.readFile("test/res/lottie-bee.json", "utf8") + const resultStream = await convert(input) + /* c8 ignore next 3 */ + if (WRITE_PNG) { + resultStream.pipe(fs.createWriteStream("test/res/lottie-bee.png")) + t.fail("PNG written to /test/res/lottie-bee.png, please manually check it") + } else { + const expected = await fs.promises.readFile("test/res/lottie-bee.png") + const actual = Buffer.alloc(expected.length) + let i = 0 + await stream.promises.pipeline( + resultStream, + async function* (source) { + for await (const chunk of source) { + chunk.copy(actual, i) + i += chunk.length + } + }, + new stream.PassThrough() + ) + t.equal(i, actual.length, `allocated ${actual.length} bytes, but wrote ${i}`) + t.deepEqual(actual, expected) + } +}) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index cb9e900..5c39853 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -12,8 +12,8 @@ const {sync, db, discord, select, from} = passthrough const file = sync.require("../../matrix/file") /** @type {import("./emoji-to-key")} */ const emojiToKey = sync.require("./emoji-to-key") -/** @type {import("./lottie")} */ -const lottie = sync.require("./lottie") +/** @type {import("../actions/lottie")} */ +const lottie = sync.require("../actions/lottie") /** @type {import("../../m2d/converters/utils")} */ const mxUtils = sync.require("../../m2d/converters/utils") /** @type {import("../../discord/utils")} */ diff --git a/test/res/lottie-bee.json b/test/res/lottie-bee.json new file mode 100644 index 0000000..ebf9c08 --- /dev/null +++ b/test/res/lottie-bee.json @@ -0,0 +1 @@ +{"v":"4.8.0","meta":{"g":"LottieFiles AE 3.5.3","a":"","k":"","d":"","tc":""},"fr":30,"ip":0,"op":30,"w":500,"h":500,"nm":"bee 2","ddd":0,"assets":[{"id":"comp_0","layers":[{"ddd":0,"ind":1,"ty":4,"nm":"face Outlines","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":0,"s":[-4]},{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":5,"s":[0]},{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":10,"s":[-4]},{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":15,"s":[0]},{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":20,"s":[-4]},{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":25,"s":[0]},{"t":30,"s":[-4]}],"ix":10},"p":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[253.272,244.9,0],"to":[-0.026,0.26,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":5,"s":[253.115,246.462,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":10,"s":[253.272,244.9,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[253.115,246.462,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":20,"s":[253.272,244.9,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":25,"s":[253.115,246.462,0],"to":[0,0,0],"ti":[-0.026,0.26,0]},{"t":30,"s":[253.272,244.9,0]}],"ix":2},"a":{"a":0,"k":[5.561,37.055,0],"ix":1},"s":{"a":1,"k":[{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":0,"s":[97.459,100.907,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":5,"s":[100,100,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":10,"s":[97.459,100.907,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":15,"s":[100,100,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":20,"s":[97.459,100.907,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":25,"s":[100,100,100]},{"t":30,"s":[97.459,100.907,100]}],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[4.453,5.599]],"o":[[-1.909,4.708],[0,0]],"v":[[7.444,-2.8],[-7.444,-2.8]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":2,"lj":2,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[50.761,57.811],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[-4.337,-9.246]],"o":[[4.155,-6.455],[0,0]],"v":[[-11.383,3.198],[8.852,3.974]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":2,"lj":2,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[72.537,12.212],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 2","np":2,"cix":2,"bm":0,"ix":2,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[7.088,-9.246]],"o":[[-6.791,-6.455],[0,0]],"v":[[13.215,1.743],[-13.215,4.712]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":2,"lj":2,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[20.715,12.212],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 3","np":2,"cix":2,"bm":0,"ix":3,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-1.927,0],[0,2.63],[1.927,0],[0,-2.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":3,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.396,-0.25],[-0.563,2.567],[1.302,-0.25],[-0.031,-2.755]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":6,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.896,-0.688],[-0.625,2.005],[1.802,-0.313],[-0.281,-1.88]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":9,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-1.927,0],[0,2.63],[1.927,0],[0,-2.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":12,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.396,-0.25],[-0.563,2.567],[1.302,-0.25],[-0.031,-2.755]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.896,-0.688],[-0.625,2.005],[1.802,-0.313],[-0.281,-1.88]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":18,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-1.927,0],[0,2.63],[1.927,0],[0,-2.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":21,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.396,-0.25],[-0.563,2.567],[1.302,-0.25],[-0.031,-2.755]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":24,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.896,-0.688],[-0.625,2.005],[1.802,-0.313],[-0.281,-1.88]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":27,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.396,-0.25],[-0.563,2.567],[1.302,-0.25],[-0.031,-2.755]],"c":true}]},{"t":30,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-1.927,0],[0,2.63],[1.927,0],[0,-2.63]],"c":true}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.427451010311,0.427451010311,0.427451010311,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[68.507,21.95],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 4","np":2,"cix":2,"bm":0,"ix":4,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.698],[1.077,5.632],[1.958,-0.698],[-1.079,-5.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":3,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.083,0.573],[1.327,4.757],[2.333,-1.198],[-0.454,-4.38]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":6,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.073],[1.452,3.757],[2.583,-1.573],[-0.079,-2.88]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":9,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.698],[1.077,5.632],[1.958,-0.698],[-1.079,-5.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":12,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.083,0.573],[1.327,4.757],[2.333,-1.198],[-0.454,-4.38]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.073],[1.452,3.757],[2.583,-1.573],[-0.079,-2.88]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":18,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.698],[1.077,5.632],[1.958,-0.698],[-1.079,-5.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":21,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.083,0.573],[1.327,4.757],[2.333,-1.198],[-0.454,-4.38]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":24,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.073],[1.452,3.757],[2.583,-1.573],[-0.079,-2.88]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":27,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.083,0.573],[1.327,4.757],[2.333,-1.198],[-0.454,-4.38]],"c":true}]},{"t":30,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.698],[1.077,5.632],[1.958,-0.698],[-1.079,-5.63]],"c":true}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[1,1,1,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[77.599,28.9],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 5","np":2,"cix":2,"bm":0,"ix":5,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,-9.389],[-6.879,0],[0,9.389],[6.878,0]],"o":[[0,9.389],[6.878,0],[0,-9.389],[-6.879,0]],"v":[[-12.455,0],[0,17],[12.455,0],[0,-17]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[71.804,34.72],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 6","np":2,"cix":2,"bm":0,"ix":6,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-1.927,0],[0,2.63],[1.927,0],[0,-2.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":3,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.177,-0.188],[-0.563,2.755],[1.364,-0.125],[0.5,-2.692]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":6,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.489,-0.625],[-0.5,2.067],[1.677,-0.625],[-0.25,-2.567]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":9,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-1.927,0],[0,2.63],[1.927,0],[0,-2.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":12,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.177,-0.188],[-0.563,2.755],[1.364,-0.125],[0.5,-2.692]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.489,-0.625],[-0.5,2.067],[1.677,-0.625],[-0.25,-2.567]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":18,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-1.927,0],[0,2.63],[1.927,0],[0,-2.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":21,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.177,-0.188],[-0.563,2.755],[1.364,-0.125],[0.5,-2.692]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":24,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.489,-0.625],[-0.5,2.067],[1.677,-0.625],[-0.25,-2.567]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":27,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-2.177,-0.188],[-0.563,2.755],[1.364,-0.125],[0.5,-2.692]],"c":true}]},{"t":30,"s":[{"i":[[0,-1.453],[-1.064,0],[0,1.453],[1.065,0]],"o":[[0,1.453],[1.065,0],[0,-1.453],[-1.064,0]],"v":[[-1.927,0],[0,2.63],[1.927,0],[0,-2.63]],"c":true}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.427451010311,0.427451010311,0.427451010311,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[20.181,21.95],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 7","np":2,"cix":2,"bm":0,"ix":7,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.698],[1.077,5.632],[1.958,-0.698],[-1.079,-5.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":3,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.333,0.448],[1.202,4.507],[2.583,-0.323],[-0.829,-4.88]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":6,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.833,-0.552],[1.202,3.132],[2.458,-1.448],[-0.704,-4.13]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":9,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.698],[1.077,5.632],[1.958,-0.698],[-1.079,-5.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":12,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.333,0.448],[1.202,4.507],[2.583,-0.323],[-0.829,-4.88]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.833,-0.552],[1.202,3.132],[2.458,-1.448],[-0.704,-4.13]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":18,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.698],[1.077,5.632],[1.958,-0.698],[-1.079,-5.63]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":21,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.333,0.448],[1.202,4.507],[2.583,-0.323],[-0.829,-4.88]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":24,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.833,-0.552],[1.202,3.132],[2.458,-1.448],[-0.704,-4.13]],"c":true}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":27,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.333,0.448],[1.202,4.507],[2.583,-0.323],[-0.829,-4.88]],"c":true}]},{"t":30,"s":[{"i":[[-0.595,-3.11],[-1.081,0.385],[0.595,3.11],[1.082,-0.386]],"o":[[0.595,3.11],[1.082,-0.386],[-0.596,-3.11],[-1.081,0.385]],"v":[[-1.958,0.698],[1.077,5.632],[1.958,-0.698],[-1.079,-5.63]],"c":true}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[1,1,1,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[29.273,28.9],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 8","np":2,"cix":2,"bm":0,"ix":8,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,-9.389],[-6.879,0],[0,9.389],[6.878,0]],"o":[[0,9.389],[6.878,0],[0,-9.389],[-6.879,0]],"v":[[-12.455,0],[0,17],[12.455,0],[0,-17]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[23.478,34.72],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 9","np":2,"cix":2,"bm":0,"ix":9,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,-4.624],[-4.624,0],[0,4.624],[4.624,0]],"o":[[0,4.624],[4.624,0],[0,-4.624],[-4.624,0]],"v":[[-8.373,0],[0,8.373],[8.373,0],[0,-8.373]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.956862804936,0.337254901961,0.36862745098,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[17.1,53.555],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 10","np":2,"cix":2,"bm":0,"ix":10,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,-3.547],[-2.943,0],[0,3.547],[2.942,0]],"o":[[0,3.547],[2.942,0],[0,-3.547],[-2.943,0]],"v":[[-5.328,0],[0,6.423],[5.328,0],[0,-6.423]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.956862804936,0.337254901961,0.36862745098,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[71.804,54.649],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 11","np":2,"cix":2,"bm":0,"ix":11,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":30,"st":0,"bm":0},{"ddd":0,"ind":2,"ty":4,"nm":"Layer 2 Outlines","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[245.871,249.996,0],"ix":2},"a":{"a":0,"k":[102.101,92.65,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,-26.818],[-26.818,0],[0,26.818],[26.818,0]],"o":[[0,26.818],[26.818,0],[0,-26.818],[-26.818,0]],"v":[[-48.558,0],[0,48.558],[48.558,0],[0,-48.558]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":2,"lj":2,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.976470648074,0.768627510819,0.352941176471,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[141.819,133.219],"ix":2},"a":{"a":0,"k":[-1,49],"ix":1},"s":{"a":1,"k":[{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":0,"s":[97.876,103.587]},{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":5,"s":[101.984,98.447]},{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":10,"s":[97.876,103.587]},{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":15,"s":[101.984,98.447]},{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":20,"s":[97.876,103.587]},{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":25,"s":[101.984,98.447]},{"t":30,"s":[97.876,103.587]}],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[2.186,-8.674],[-25.374,-6.394],[-2.185,8.674],[25.374,6.393]],"o":[[-2.186,8.674],[25.373,6.393],[2.186,-8.673],[-25.374,-6.393]],"v":[[-46.242,-6.842],[-4.852,22.807],[46.242,13.943],[6.642,-22.807]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":1,"lj":1,"ml":10,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.717647058824,0.901960844152,0.917647118662,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[101.677,77.54],"ix":2},"a":{"a":0,"k":[45.75,13.5],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":1,"k":[{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":1,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":3,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":5,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":7,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":9,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":11,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":13,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":16,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":18,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":20,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":22,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":25,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":27,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":29,"s":[9]},{"t":33,"s":[0]}],"ix":6},"o":{"a":0,"k":50,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 12","np":3,"cix":2,"bm":0,"ix":2,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[2.186,-8.674],[-25.374,-6.394],[-2.185,8.674],[25.374,6.393]],"o":[[-2.186,8.674],[25.373,6.393],[2.186,-8.673],[-25.374,-6.393]],"v":[[-46.242,-6.842],[-4.852,22.807],[46.242,13.943],[6.642,-22.807]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":1,"lj":1,"ml":10,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.717647058824,0.901960844152,0.917647118662,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[101.677,77.54],"ix":2},"a":{"a":0,"k":[45.75,13.5],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":1,"k":[{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":0,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":2,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":4,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":6,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":8,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":10,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":12,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":14,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":17,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":19,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":21,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":24,"s":[9]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":26,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":28,"s":[9]},{"t":30,"s":[0]}],"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 2","np":3,"cix":2,"bm":0,"ix":3,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[-6.474,1.689],[17.644,24.6],[0,0],[2.299,-5.075],[-13.742,-20.189]],"o":[[6.214,0.149],[0,0],[-16.495,-22.997],[-3.916,4.513],[0,0],[12.03,17.673]],"v":[[11.746,37.811],[30.898,35.531],[-2.929,4.055],[-21.525,-37.961],[-30.898,-23.482],[-16.162,14.745]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[65.896,132.826],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 3","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[6.191,-2.121],[-10.084,-18.006],[0,0],[-3.996,4.256],[14.081,22.475]],"o":[[-6.081,0.401],[0,0],[16.833,30.055],[5.298,-3.32],[0,0],[-10.838,-17.299]],"v":[[-9.343,-44.503],[-27.856,-40.746],[-18.074,5.609],[14.172,44.503],[28.158,33.051],[-1.24,-3.71]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[104.236,113.47],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 4","np":2,"cix":2,"bm":0,"ix":2,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[-10.845,-24.194],[-30.325,13.593],[10.845,24.194],[30.325,-13.594]],"o":[[10.845,24.194],[30.325,-13.593],[-10.845,-24.194],[-30.325,13.593]],"v":[[-54.909,24.613],[19.637,43.807],[54.909,-24.613],[-19.637,-43.807]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":2,"lj":2,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.976470648074,0.768627510819,0.352941176471,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[88.928,120.399],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 5","np":3,"cix":2,"bm":0,"ix":3,"mn":"ADBE Vector Group","hd":false},{"ty":"tr","p":{"a":0,"k":[88.928,170.312],"ix":2},"a":{"a":0,"k":[88.928,170.312],"ix":1},"s":{"a":1,"k":[{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":0,"s":[96.599,101.49]},{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":5,"s":[100.843,98.022]},{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":10,"s":[96.599,101.49]},{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":15,"s":[100.843,98.022]},{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":20,"s":[96.599,101.49]},{"i":{"x":[0.667,0.667],"y":[1,1]},"o":{"x":[0.333,0.333],"y":[0,0]},"t":25,"s":[100.843,98.022]},{"t":30,"s":[96.599,101.49]}],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 14","np":3,"cix":2,"bm":0,"ix":4,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[5.677,-6.912],[-20.22,-16.608],[-5.677,6.912],[20.22,16.608]],"o":[[-5.677,6.912],[20.221,16.608],[5.678,-6.913],[-20.221,-16.608]],"v":[[-37.387,-28.343],[-12.605,16.131],[37.387,29.915],[17.253,-20.219]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":1,"lj":1,"ml":10,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.596078431373,0.776470648074,0.800000059838,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[101.655,79.493],"ix":2},"a":{"a":0,"k":[36.75,29.75],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":1,"k":[{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":0,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":2,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":4,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":6,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":8,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":10,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":12,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":14,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":17,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":19,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":21,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":24,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":26,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":28,"s":[7]},{"t":30,"s":[0]}],"ix":6},"o":{"a":0,"k":50,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 13","np":3,"cix":2,"bm":0,"ix":5,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[5.677,-6.912],[-20.22,-16.608],[-5.677,6.912],[20.22,16.608]],"o":[[-5.677,6.912],[20.221,16.608],[5.678,-6.913],[-20.221,-16.608]],"v":[[-37.387,-28.343],[-12.605,16.131],[37.387,29.915],[17.253,-20.219]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":1,"lj":1,"ml":10,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.596078431373,0.776470648074,0.800000059838,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[101.655,79.493],"ix":2},"a":{"a":0,"k":[36.75,29.75],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":1,"k":[{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":1,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":3,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":5,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":7,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":9,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":11,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":13,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":16,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":18,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":20,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":22,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":25,"s":[7]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.167],"y":[0]},"t":27,"s":[0]},{"i":{"x":[0],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":29,"s":[7]},{"t":33,"s":[0]}],"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 6","np":3,"cix":2,"bm":0,"ix":6,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,-3.884],[-3.884,0],[0,3.884],[3.884,0]],"o":[[0,3.884],[3.884,0],[0,-3.884],[-3.884,0]],"v":[[-7.032,0],[0,7.032],[7.032,0],[0,-7.032]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.16862745098,0.16862745098,0.16862745098,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[158.769,16.462],"to":[0.312,1.042],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":5,"s":[160.644,22.712],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":10,"s":[158.769,16.462],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[160.644,22.712],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":20,"s":[158.769,16.462],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":25,"s":[160.644,22.712],"to":[0,0],"ti":[0.312,1.042]},{"t":30,"s":[158.769,16.462]}],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 7","np":2,"cix":2,"bm":0,"ix":7,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[{"i":[[0,0],[-21.885,-8.483]],"o":[[0.356,-17.69],[0,0]],"v":[[-16.243,21.084],[16.243,-12.601]],"c":false}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":5,"s":[{"i":[[0,0],[-18.701,-12.079]],"o":[[0.356,-17.69],[0,0]],"v":[[-16.243,21.084],[20.493,-5.851]],"c":false}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":10,"s":[{"i":[[0,0],[-21.885,-8.483]],"o":[[0.356,-17.69],[0,0]],"v":[[-16.243,21.084],[16.243,-12.601]],"c":false}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[{"i":[[0,0],[-18.701,-12.079]],"o":[[0.356,-17.69],[0,0]],"v":[[-16.243,21.084],[20.493,-5.851]],"c":false}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":20,"s":[{"i":[[0,0],[-21.885,-8.483]],"o":[[0.356,-17.69],[0,0]],"v":[[-16.243,21.084],[16.243,-12.601]],"c":false}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":25,"s":[{"i":[[0,0],[-18.701,-12.079]],"o":[[0.356,-17.69],[0,0]],"v":[[-16.243,21.084],[20.493,-5.851]],"c":false}]},{"t":30,"s":[{"i":[[0,0],[-21.885,-8.483]],"o":[[0.356,-17.69],[0,0]],"v":[[-16.243,21.084],[16.243,-12.601]],"c":false}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":2,"lj":2,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[140.438,28.584],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 8","np":2,"cix":2,"bm":0,"ix":8,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,-3.884],[-3.884,0],[0,3.884],[3.884,0]],"o":[[0,3.884],[3.884,0],[0,-3.884],[-3.884,0]],"v":[[-7.032,0],[0,7.032],[7.032,0],[0,-7.032]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"fl","c":{"a":0,"k":[0.16862745098,0.16862745098,0.16862745098,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[194.544,28.445],"to":[0.083,0.75],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":5,"s":[195.044,32.945],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":10,"s":[194.544,28.445],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[195.044,32.945],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":20,"s":[194.544,28.445],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":25,"s":[195.044,32.945],"to":[0,0],"ti":[0.083,0.75]},{"t":30,"s":[194.544,28.445]}],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 9","np":2,"cix":2,"bm":0,"ix":9,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[{"i":[[0,0],[-13.371,-15.777]],"o":[[1.393,-16.89],[0,0]],"v":[[-17.035,18.539],[17.035,-2.762]],"c":false}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":5,"s":[{"i":[[0,0],[-10.362,-15.651]],"o":[[1.393,-16.89],[0,0]],"v":[[-17.035,18.539],[17.535,1.925]],"c":false}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":10,"s":[{"i":[[0,0],[-13.371,-15.777]],"o":[[1.393,-16.89],[0,0]],"v":[[-17.035,18.539],[17.035,-2.762]],"c":false}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[{"i":[[0,0],[-10.362,-15.651]],"o":[[1.393,-16.89],[0,0]],"v":[[-17.035,18.539],[17.535,1.925]],"c":false}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":20,"s":[{"i":[[0,0],[-13.371,-15.777]],"o":[[1.393,-16.89],[0,0]],"v":[[-17.035,18.539],[17.035,-2.762]],"c":false}]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":25,"s":[{"i":[[0,0],[-10.362,-15.651]],"o":[[1.393,-16.89],[0,0]],"v":[[-17.035,18.539],[17.535,1.925]],"c":false}]},{"t":30,"s":[{"i":[[0,0],[-13.371,-15.777]],"o":[[1.393,-16.89],[0,0]],"v":[[-17.035,18.539],[17.035,-2.762]],"c":false}]}],"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":2,"lj":2,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[177.682,31.129],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 10","np":2,"cix":2,"bm":0,"ix":10,"mn":"ADBE Vector Group","hd":false},{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[-1.509,-3.032],[-5.652,2.813],[1.509,3.032],[5.652,-2.813]],"o":[[1.509,3.032],[5.653,-2.813],[-1.509,-3.032],[-5.653,2.813]],"v":[[-13.036,9.949],[5.172,4.341],[13.036,-7.114],[-1.858,-10.167]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.219607858097,0.219607858097,0.219607858097,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":3,"ix":5},"lc":2,"lj":2,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"fl","c":{"a":0,"k":[0.16862745098,0.16862745098,0.16862745098,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[41.787,163.688],"to":[0.146,-0.115],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":5,"s":[42.662,163.001],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":10,"s":[41.787,163.688],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[42.662,163.001],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":20,"s":[41.787,163.688],"to":[0,0],"ti":[0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":25,"s":[42.662,163.001],"to":[0,0],"ti":[0.146,-0.115]},{"t":30,"s":[41.787,163.688]}],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 11","np":3,"cix":2,"bm":0,"ix":11,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":30,"st":0,"bm":0}]}],"layers":[{"ddd":0,"ind":1,"ty":4,"nm":"line 20","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[250,250,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":18,"s":[0]},{"t":24,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":16,"s":[0]},{"t":22,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":16,"op":24,"st":16,"bm":0},{"ddd":0,"ind":2,"ty":4,"nm":"line 19","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[312.5,165.5,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":14,"s":[0]},{"t":20,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":12,"s":[0]},{"t":18,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":12,"op":20,"st":12,"bm":0},{"ddd":0,"ind":3,"ty":4,"nm":"line 18","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[186,200,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":15,"s":[0]},{"t":21,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":13,"s":[0]},{"t":19,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":13,"op":21,"st":13,"bm":0},{"ddd":0,"ind":4,"ty":4,"nm":"line 17","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[312.5,286,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":17,"s":[0]},{"t":23,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":15,"s":[0]},{"t":21,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":15,"op":23,"st":15,"bm":0},{"ddd":0,"ind":5,"ty":4,"nm":"line 16","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[187.5,316,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":16,"s":[0]},{"t":22,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":14,"s":[0]},{"t":20,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":14,"op":22,"st":14,"bm":0},{"ddd":0,"ind":6,"ty":4,"nm":"line 15","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[187.5,316,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":22,"s":[0]},{"t":28,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":20,"s":[0]},{"t":26,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":20,"op":28,"st":20,"bm":0},{"ddd":0,"ind":7,"ty":4,"nm":"line 14","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[312.5,286,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":23,"s":[0]},{"t":29,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":21,"s":[0]},{"t":27,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":21,"op":29,"st":21,"bm":0},{"ddd":0,"ind":8,"ty":4,"nm":"line 13","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[186,200,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":21,"s":[0]},{"t":27,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":19,"s":[0]},{"t":25,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":19,"op":27,"st":19,"bm":0},{"ddd":0,"ind":9,"ty":4,"nm":"line 12","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[312.5,165.5,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":20,"s":[0]},{"t":26,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":18,"s":[0]},{"t":24,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":18,"op":26,"st":18,"bm":0},{"ddd":0,"ind":10,"ty":4,"nm":"line 11","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[250,250,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":24,"s":[0]},{"t":30,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":22,"s":[0]},{"t":28,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":22,"op":30,"st":22,"bm":0},{"ddd":0,"ind":11,"ty":4,"nm":"line 10","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[250,250,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":13,"s":[0]},{"t":19,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":11,"s":[0]},{"t":17,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":11,"op":19,"st":11,"bm":0},{"ddd":0,"ind":12,"ty":4,"nm":"line 9","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[312.5,165.5,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":9,"s":[0]},{"t":15,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":7,"s":[0]},{"t":13,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":7,"op":15,"st":7,"bm":0},{"ddd":0,"ind":13,"ty":4,"nm":"line 8","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[186,200,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":10,"s":[0]},{"t":16,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":8,"s":[0]},{"t":14,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":8,"op":16,"st":8,"bm":0},{"ddd":0,"ind":14,"ty":4,"nm":"line 7","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[312.5,286,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":12,"s":[0]},{"t":18,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":10,"s":[0]},{"t":16,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":10,"op":18,"st":10,"bm":0},{"ddd":0,"ind":15,"ty":4,"nm":"line 6","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[187.5,316,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":11,"s":[0]},{"t":17,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":9,"s":[0]},{"t":15,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":9,"op":17,"st":9,"bm":0},{"ddd":0,"ind":16,"ty":4,"nm":"line 5","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[187.5,316,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":5,"s":[0]},{"t":11,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":3,"s":[0]},{"t":9,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":3,"op":11,"st":3,"bm":0},{"ddd":0,"ind":17,"ty":4,"nm":"line 4","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[312.5,286,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":6,"s":[0]},{"t":12,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":4,"s":[0]},{"t":10,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":4,"op":12,"st":4,"bm":0},{"ddd":0,"ind":18,"ty":4,"nm":"line 3","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[186,200,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":4,"s":[0]},{"t":10,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":2,"s":[0]},{"t":8,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":2,"op":10,"st":2,"bm":0},{"ddd":0,"ind":19,"ty":4,"nm":"line 2","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[312.5,165.5,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":3,"s":[0]},{"t":9,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":1,"s":[0]},{"t":7,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":1,"op":9,"st":1,"bm":0},{"ddd":0,"ind":20,"ty":4,"nm":"line","sr":1,"ks":{"o":{"a":0,"k":30,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[250,250,0],"ix":2},"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,0],[0,0]],"o":[[0,0],[0,0]],"v":[[63,0.5],[-62,0.5]],"c":false},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0,0,0,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":4,"ix":5},"lc":1,"lj":1,"ml":4,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[0,0],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Shape 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":7,"s":[0]},{"t":13,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":5,"s":[0]},{"t":11,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":5,"op":13,"st":5,"bm":0},{"ddd":0,"ind":21,"ty":4,"nm":"Shape Layer 1","sr":1,"ks":{"o":{"a":0,"k":50,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":0,"k":[250,404,0],"ix":2},"a":{"a":0,"k":[1,154,0],"ix":1},"s":{"a":1,"k":[{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":0,"s":[95,95,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":5,"s":[100,100,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":10,"s":[95,95,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":15,"s":[100,100,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":20,"s":[95,95,100]},{"i":{"x":[0.667,0.667,0.667],"y":[1,1,1]},"o":{"x":[0.333,0.333,0.333],"y":[0,0,0]},"t":25,"s":[100,100,100]},{"t":30,"s":[95,95,100]}],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"d":1,"ty":"el","s":{"a":0,"k":[110,26],"ix":2},"p":{"a":0,"k":[0,0],"ix":3},"nm":"Ellipse Path 1","mn":"ADBE Vector Shape - Ellipse","hd":false},{"ty":"fl","c":{"a":0,"k":[0.235294117647,0.235294117647,0.235294117647,1],"ix":4},"o":{"a":0,"k":100,"ix":5},"r":1,"bm":0,"nm":"Fill 1","mn":"ADBE Vector Graphic - Fill","hd":false},{"ty":"tr","p":{"a":0,"k":[1,154],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Ellipse 1","np":3,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false}],"ip":0,"op":30,"st":0,"bm":0},{"ddd":0,"ind":22,"ty":0,"nm":"bee","refId":"comp_0","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":0,"k":0,"ix":10},"p":{"a":1,"k":[{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":0,"s":[250,250,0],"to":[0,1.167,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":5,"s":[250,257,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":10,"s":[250,250,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":15,"s":[250,257,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":20,"s":[250,250,0],"to":[0,0,0],"ti":[0,0,0]},{"i":{"x":0.667,"y":1},"o":{"x":0.333,"y":0},"t":25,"s":[250,257,0],"to":[0,0,0],"ti":[0,1.167,0]},{"t":30,"s":[250,250,0]}],"ix":2},"a":{"a":0,"k":[250,250,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"w":500,"h":500,"ip":0,"op":30,"st":0,"bm":0}],"markers":[]} \ No newline at end of file diff --git a/test/res/lottie-bee.license.txt b/test/res/lottie-bee.license.txt new file mode 100644 index 0000000..953aeb4 --- /dev/null +++ b/test/res/lottie-bee.license.txt @@ -0,0 +1,16 @@ +# Source + +Flying Bee by Afif Ridwan +https://lottiefiles.com/animations/flying-bee-WkXvUiWkZ1 + +# License + +Lottie Simple License (FL 9.13.21) https://lottiefiles.com/page/license + +Copyright © 2021 Design Barn Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of the public animation files available for download at the LottieFiles site (“Files”) to download, reproduce, modify, publish, distribute, publicly display, and publicly digitally perform such Files, including for commercial purposes, provided that any display, publication, performance, or distribution of Files must contain (and be subject to) the same terms and conditions of this license. Modifications to Files are deemed derivative works and must also be expressly distributed under the same terms and conditions of this license. You may not purport to impose any additional or different terms or conditions on, or apply any technical measures that restrict exercise of, the rights granted under this license. This license does not include the right to collect or compile Files from LottieFiles to replicate or develop a similar or competing service. + +Use of Files without attributing the creator(s) of the Files is permitted under this license, though attribution is strongly encouraged. If attributions are included, such attributions should be visible to the end user. + +FILES ARE PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL THE CREATOR(S) OF FILES OR DESIGN BARN, INC. BE LIABLE ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE, OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF SUCH FILES. diff --git a/test/res/lottie-bee.png b/test/res/lottie-bee.png new file mode 100644 index 0000000000000000000000000000000000000000..3fdc19a99dd727d13647e9bda65d547daee86775 GIT binary patch literal 5492 zcmcIo_ct6s*Is2=CF+tzw6(g(N_0sGt0j6DC2B-O5W$KTtP)*>AWMP}y+)6)+7g|t zmMp7>CCWx`-@flZ@SX36Z_eDgXU>^(pE>u=%$@tZF*4A;LeEVP006G&>L5(W`qO^| zL_;3qN8GNH6{RlXo*9y2lRI=A0N}X(pV=0ME;!vS=f=+bW#IfF8Z8%Oz(zTvqduM? z6$4X?bB()?m+-_%TeRUCo@!&WlKiguq3yLj{Iqf`u{hzsefkV^zO(VBAoH~2MBo>NHCYUzTV6kpFrm7+dA~slTXjitmb2; zWNrJS2>J!0hN1cG?O}lTu~WnfWAIAYOS{YRk+sw7!xGo8Z<(3<3EBGq09dF7yaeYl z6+MLyE;BJREh9kn7<1q)_P?Pt%$shAKUuq{RVv6Gg7G&BNsID6pH}=(wBzq(((!>) zZf$Mtr6%GdB|M*avUYlMlC)TEUS`pjBjeA&S^-u^-()Nh2Ges;^~o_VrOwUGVI3%` z*-MLyi|-E%K0V$Exhy5f(E}-?w-wsK*n|3;LVWF2_4WH7ca?EC+#L$SN=8NoKLMJa zp5FX6EsY;ojR5Ulx0^bY>8`(!vmEcxyRH*tCSNa*!B)pRQ=fT|+#>t>Vq zV9EQIRdU??fk*sP<(^IJE35bwfTBHB)szhtVUKJ%nE>VB;1Cx5)}eIiYGd>33bIEn zFffpYbs;&nm`Ux{+@e{g5ji@C+0ittnc1@F5_fugT0_Uwz3{4w_?nd%X3p~G9izGU zRt7+~cPd!3Imh?n--hz`T8{q~eQ&Xv_kv))HH8n9`)V7e`a5ooR{>y1^llwF9G-BB2wihaORs7byOH_5L@$QnX2qExkWh%W z*z!l3Z~^L=p_S+_O;*eAD%IW&RXS0O+}ZIuSK06v8%KA&0oW39z*R2@#FKWH-M)sHUw28d3!|nMnNIU@JQd(B# z^;IQjo`N7=S-xs%Tc7)bXc;LU7XiGcT0Otx8JK%A8`5p2U;b)d#D25FTn1ahPSs1A zzvL_WQk3Br5ReRYJl96*E5FLe-RcgEMI8hLCUgpBw<5KQIF5G?0Z}p>1Kor^WD~x zt|f~So5%ADFTb{Q`rz;iF3R3?g`#C<4L_Y`@a!|6izoRF_4PF!P3ZML6*o17Z50vI z4?{Mp%*}BSX$nfK1rAAT>n(}7f27NTYn~n}!evu0hMdgLmz<-+UOuz)&mQ0L3RqP% zj{?2#>p;Ha=S_Mu{tadYu4ObV@x#(a^Tp_jBjZW0+8zyB)w*;gUhW8Ucu~6Dmmq!j z722LYfT~o7NTyf1!%ciMVlO6ha!*xrJ@-F9W%NZWa;)%%u*xX%v6s>>G)SW2NByJ* z+t2!nk?vtEcu}XLeo~s1^)olDR=#!8a(RxBkdOvk5a{JYzRpu_SHU=oa~zI-4pv$! zv#lb(7uouu*zHhyI?Yr}7)jmuT*usdGJMJLB3&EXeZzl5AfMqMI(9M3?$-1?Kj(n2 zTfOio+bq~`Xc4^n(HH+AROT3!EqH4Y4R_@n(_-2VQ4 zY<_r$o5$R)!SnhV$Q-O2V$=Lr;$p8V4AaY8zf`91#WdSS7`C}VzvK1nZdCisyQKD3 zL3X00^31O?yO$x6HrnOG#Zi9F?MF*mGvhFM_M1lw?8d3NA52pTYx=ZhUCvjuIQ49L{#S$bi=XywMqj-Sb z2TTfFxsVigvJUy~7bGtczm>q&DKi{n(1VE6xTkB+M#`9`V-V|WFln(VjQRL+jQ{L9 z_EuuSk5%4@{Eh-gzvf7w5Dk$lK_3lKZl)3`=%9`^KqH}&lEg_W{Z^A3?R%c;bq~4b zVA``GV)^#v;*GapR&s2XmMW>+HE`IO3$gqH{|LK##8B{~PCvBfwl;fU7H22$+5XTD ze;doEgao~8H2U+3aN%M{HOEZSYpWV5-I-awKBYO;Gk!Q1P1=*t^}^cqv9VQJRg(Ay zaoUZbfAeSad+r|^Z@&>FRY9-GpRz_5Vj2>(dTrlv2OeFDJ26}&iL`zm}1wp^YioK($Y>s zi6RdqSr9+u#Fy+wrx%TJ3BEhum~^ne!s~wM7jk&;>yLu!*>^tlE>9EUana+HRSTJU z0`eVm0+G>|k%=k4sE7t1DI+gm;fu!p)VoHiZ)jK~U?LR@f&}6i8!5|h*;mqx9o3zk zotMZ`FKJo%?VIn^u&6S3qvdjX`ntz_7;pOIWF!DsWh%xJF_3|Qk1UsmACo^W-b;xJ ziwS=(;J3}C(GPxiG*`E7f9G*SIbX{K>jOJy;9MLclQ^6oK+QEYH1z&$&eMawF=qvz z=hJ)l2mZy?Uok@--*=I=Dhrb?cI~D%w6_nbPm-ZY~DbWqrmXd~lbcAeHw*Kwp z!o+qQgV!Dkq6mm?Z>D~KV0f&f_b?S1BEEy%{*fgI@b}wh?LeHfUCun0(yL{=2;Uxp zZIh{qqi?H2Kr^c6aSUs+Mlx0=WJ4DC>y;>^MQN|d<12fgi!;lRZ#Woy ze$lNh={S{3jdTXZEM*#kFaz(ccLt@0-#nofDR4?1lO5Xd7bEhWd|-67&PX4kUV3HY zMO`ovnqZhyq=xEx>{vbhdY(5|4$69uPC|iEj4r`J?__C*Y+n*zSo>Qx{UiD(m%ZlB zc%i)1@_~oRZWMpA^IfkuS03Cff99lRsh&3q($ngJniw0WZuviK2qQIYF1uP=L{m3v zm{wIzdP_%sS^W7zu@kN>Ak~e*Kz@)sGq;y`On3Ib7&w1#xye&&Y90;z`Cet_dKkxw zq5KKg@cjUpi=Gd)a2|Q2fRj-`z@gZG5ZnzbG|pQ09vKjRPs<} z>7bnyUNZFFDKmtTEP}u`>UE56g{_dwQ)U-}Q}EuM_3~cN37ZB@tJ2cNY<;HOjQrwX zvu@2ry%=N%`5w3|$wuX)m!y8MW+uO&UbT7a z=sJJ;lIh_qjiXs-XJoS|%4=@*Udsblollf%?r=pCH@af^Xlz>~B3ke-KQ}jh7s^bE zQkQeTZcGu3u0J#a-4cv!-J$RrZ2Ir6FxMO9C`^p!zvH} zg=--|l0)(RZ&`-Zy&*m{t+RdjtrKj1@YIFTX5nfEt)mH}$lCrZ;!RVqq=4e^V#M@h zNL>?TyCUEo2qR-CFgW9NFgQAw3&2|cP#*r;x;T2*^R-3BCmv@8( zApUEpx|Z3#U0%u6x+uCi*E9PbB^D?(P+N;z@gLMiJdCYP@_0>NXVS{qP58ZAA(0a?^5F1 zMa;3YCOr_~zIwKO83aRm3w-l+Gab0<_<~X7>MbLSOlQrIEMC&N%K(^7QJN5^&STR) zKJ3GE!7@c(D|N=t&(%{|`8r9#;dd2Dr9A*TdW$9GtIyh;M-2#`$-@=x-LyqtcD+u+ zv_nRF+Z(;LK~Fx+Cn4^mX5MIx2VPcfyPo}L{adD+xiN-&m|Zcf2TLNuMZvrw;_kuL zy~|OTv%kIX4}zKEQIu%|k1`GmlfU0FH|59=-_2B%=N29GL5fMtwf9~JYoUOOp7dO{ zu}KB2ol2Xke3$^$tN}a|aQ@W^c#FuufDwE#E|*t=$kYa54*8Q~u?_mP+hMP6`#%xe zJ0k3K2dStWZeyVNo&Wk+)V-lj+a&DBoi5m#Bf#P@8xiqq_aB-Fa@pX~HbbV?o=X7qW^B{v|tSzeO0REoWpf;&6J@ zqLem&+yTED0a;bLTU2){pG>L>k>#m(^|!EjjO99amN`#jW(2m-s`EQ1T){5w=2cc! z`sf%ryyyW2`bF_Q&~!{nPa!ml!2b*(A5tnf5*^==0;7TUVHL zs`5LQZKbD1^~~DVq9xPJ^$S(ShH>4#L2f`{hos1J)*7c0O{t-;ESzd|WdmDI+HPvp zoe|zlLwLc3+SFab4bihD(}hHG_GC}7*9`C1*VF5!XV`nuWxNzOdov~)H3&{LIQH{# zQPj$eWXoov*6ebXPu(z~W2kOHd3t%x)2^ezQk0W0 zw~2o7m0LWeb{6jwS=qh`%a2WSvD66~WNNH^TK$_cZL|dTmlTrf@U-k}y{-Ffn=!VD zsq>mNPgLi^k*LkXkdb@JH-PbKj*gBcf;$UsF%bueWxuP75vA&8=4e>M5v96I0HPi`W=+m(%Y3LToW}?xAiRhSODkLbwV=s2wDo%5wzX||F8l8IL(4Rkl zzLRGpCd49N{Y6xr$HPwDG%0Uqmw8X;gmZYd-%TRgwpdVhF=-fol%|Cw>Y1Z?_;4)QH zr+m;8Js4Y!DJ9!t3#RC?vzD_PIJ`LTk$nmRIAdX9`I!BBj0P0Kic*STy=N0lQ6pu_ zam**V!11l^2JHLk=6co_?k`#f$B7?4WZQgKQdZ`UcsT|Vv9Pfr#v`EV(T?zM5x<@_ zSOb3HnfAd~UruDd_%Qu-a!X?8$r){Xu8YkfkZahnV9MRrl*r=QmU`qH^mvI<@iy-$ z{}WHo5c=1z|7El4kqfd?#KhOOaJ3G7gjBeVRED*=c^8`0`fGMpUPw??RaN}fEmCBO z^nP7xrd8@Q%f!@^x$xisiDkqGF2dNcHSTm@J;u<}x z4>Zna*cky{LStjadX0SafIjx9scF*f)+Wj zmQ_{VXna`F$YRV1=={hm?80VTk6q$Y4cuF%<`j?COpvjFp&X7F1fh@XAI4Y5%1AlY zSe3JO7@tLX4<|D+GH!N2wNeQCuY6_o%UlxDS&ZrDY)#Q5p_a-3G82;Ilq#eX)=AtK zv8kARHXPWHFXoOWfuZ*)97Lhf@oESeLBu=XzgN}us4{>0g_wb4Hb}>3DHp2bvqN^Q zR0DNyCU>G1hyDb(H!~IiHHOByXLFK{gMD=VuRNs4P8fy0f-Zux0~AE2Q31M|28c>^ Hhlu|Ht3q%l literal 0 HcmV?d00001 diff --git a/test/test.js b/test/test.js index 4663b18..6c912c8 100644 --- a/test/test.js +++ b/test/test.js @@ -60,6 +60,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../d2m/actions/register-user.test") require("../d2m/converters/edit-to-changes.test") require("../d2m/converters/emoji-to-key.test") + require("../d2m/converters/lottie.test") require("../d2m/converters/message-to-event.test") require("../d2m/converters/message-to-event.embeds.test") require("../d2m/converters/pins-to-list.test") From 011889216b1116637ce4069372e61edf6bcb8637 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 18 Jan 2024 17:04:00 +1300 Subject: [PATCH 022/431] Add lottie sticker message test data --- test/data.js | 32 ++++++++++++++++++++++++++++++++ test/ooye-test-data.sql | 3 +++ 2 files changed, 35 insertions(+) diff --git a/test/data.js b/test/data.js index ed62a2d..508d628 100644 --- a/test/data.js +++ b/test/data.js @@ -1331,6 +1331,38 @@ module.exports = { name: "pomu puff" }] }, + lottie_sticker: { + id: "1106366167788044450", + type: 0, + content: "", + channel_id: "122155380120748034", + author: { + id: "113340068197859328", + username: "Cookie 🍪", + global_name: null, + display_name: null, + avatar: "b48302623a12bc7c59a71328f72ccb39", + discriminator: "7766", + public_flags: 128, + avatar_decoration: null + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2023-05-11T23:44:09.690000+00:00", + edited_timestamp: null, + flags: 0, + components: [], + sticker_items: [{ + id: "860171525772279849", + format_type: 3, + name: "8" + }] + }, message_in_thread: { type: 0, tts: false, diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index f37f599..f99d7da 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -103,6 +103,9 @@ INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES ('!maggESguZBqGBZtSnr:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), ('!CzvdIdUQXgUjDVKxeU:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'); +INSERT INTO lottie (sticker_id, mxc_url) VALUES +('860171525772279849', 'mxc://cadence.moe/ZtvvVbwMIdUZeovWVyGVFCeR'); + INSERT INTO "auto_emoji" ("name","emoji_id","guild_id") VALUES ('L1','1144820033948762203','529176156398682115'), ('L2','1144820084079087647','529176156398682115'), From d02f86b342e5075b6664b4100877fcce6fa470cb Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Jan 2024 12:37:13 +1300 Subject: [PATCH 023/431] Code coverage for unknown channel mention --- d2m/converters/message-to-event.js | 2 +- d2m/converters/message-to-event.test.js | 10 ++++++ test/data.js | 41 +++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 5c39853..0d68a73 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -43,7 +43,7 @@ function getDiscordParseCallbacks(message, guild, useHTML) { channel: node => { const row = select("channel_room", ["room_id", "name", "nick"], {channel_id: node.id}).get() if (!row) { - return `<#${node.id}>` // fallback for when this channel is not bridged + return `#[channel-from-an-unknown-server]` // fallback for when this channel is not bridged } else if (useHTML) { return `#${row.nick || row.name}` } else { diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js index 9c28f0b..b264aeb 100644 --- a/d2m/converters/message-to-event.test.js +++ b/d2m/converters/message-to-event.test.js @@ -73,6 +73,16 @@ test("message2event: simple room mention", async t => { }]) }) +test("message2event: unknown room mention", async t => { + const events = await messageToEvent(data.message.unknown_room_mention, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.text", + body: "#[channel-from-an-unknown-server]" + }]) +}) + test("message2event: simple role mentions", async t => { const events = await messageToEvent(data.message.simple_role_mentions, data.guild.general, {}) t.deepEqual(events, [{ diff --git a/test/data.js b/test/data.js index 508d628..c2c548d 100644 --- a/test/data.js +++ b/test/data.js @@ -495,6 +495,47 @@ module.exports = { attachments: [], guild_id: "112760669178241024" }, + unknown_room_mention: { + type: 0, + tts: false, + timestamp: "2023-07-10T20:04:25.939000+00:00", + referenced_message: null, + pinned: false, + nonce: "1128054139385806848", + mentions: [], + mention_roles: [], + mention_everyone: false, + member: { + roles: [], + premium_since: null, + pending: false, + nick: null, + mute: false, + joined_at: "2015-11-11T09:55:40.321000+00:00", + flags: 0, + deaf: false, + communication_disabled_until: null, + avatar: null + }, + id: "1128054143064494233", + flags: 0, + embeds: [], + edited_timestamp: null, + content: "<#555>", + components: [], + channel_id: "266767590641238027", + author: { + username: "kumaccino", + public_flags: 128, + id: "113340068197859328", + global_name: "kumaccino", + discriminator: "0", + avatar_decoration: null, + avatar: "b48302623a12bc7c59a71328f72ccb39" + }, + attachments: [], + guild_id: "112760669178241024" + }, simple_role_mentions: { id: "1162374402785153106", type: 0, From f3b7fcd1a313d725e61ee4dacea04f7298e3df2c Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Jan 2024 12:38:25 +1300 Subject: [PATCH 024/431] Full code coverage for lottie sticker --- d2m/converters/message-to-event.js | 32 +++++++------------------ d2m/converters/message-to-event.test.js | 15 ++++++++++++ 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 0d68a73..a620beb 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -497,25 +497,17 @@ async function messageToEvent(message, guild, options = {}, di) { if (message.sticker_items) { const stickerEvents = await Promise.all(message.sticker_items.map(async stickerItem => { const format = file.stickerFormat.get(stickerItem.format_type) + assert(format?.mime) if (format?.mime === "lottie") { - try { - const {mxc_url, info} = await lottie.convert(stickerItem) - return { - $type: "m.sticker", - "m.mentions": mentions, - body: stickerItem.name, - info, - url: mxc_url - } - } catch (e) { - return { - $type: "m.room.message", - "m.mentions": mentions, - msgtype: "m.notice", - body: `Failed to convert Lottie sticker:\n${e.toString()}\n${e.stack}` - } + const {mxc_url, info} = await lottie.convert(stickerItem) + return { + $type: "m.sticker", + "m.mentions": mentions, + body: stickerItem.name, + info, + url: mxc_url } - } else if (format?.mime) { + } else { let body = stickerItem.name const sticker = guild.stickers.find(sticker => sticker.id === stickerItem.id) if (sticker && sticker.description) body += ` - ${sticker.description}` @@ -529,12 +521,6 @@ async function messageToEvent(message, guild, options = {}, di) { url: await file.uploadDiscordFileToMxc(file.sticker(stickerItem)) } } - return { - $type: "m.room.message", - "m.mentions": mentions, - msgtype: "m.notice", - body: `Unsupported sticker format ${format?.mime}. Name: ${stickerItem.name}` - } })) events.push(...stickerEvents) } diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js index b264aeb..42778b6 100644 --- a/d2m/converters/message-to-event.test.js +++ b/d2m/converters/message-to-event.test.js @@ -224,6 +224,21 @@ test("message2event: stickers", async t => { }]) }) +test("message2event: lottie sticker", async t => { + const events = await messageToEvent(data.message.lottie_sticker, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.sticker", + "m.mentions": {}, + body: "8", + info: { + mimetype: "image/png", + w: 160, + h: 160 + }, + url: "mxc://cadence.moe/ZtvvVbwMIdUZeovWVyGVFCeR" + }]) +}) + test("message2event: skull webp attachment with content", async t => { const events = await messageToEvent(data.message.skull_webp_attachment_with_content, data.guild.general, {}) t.deepEqual(events, [{ From 0237e6d8fd2f2d9259920725f94495dc0356676d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Jan 2024 12:39:07 +1300 Subject: [PATCH 025/431] Improve code coverage for video --- d2m/converters/message-to-event.test.js | 35 +++++++++ test/data.js | 95 +++++++++++++++++++++++++ test/ooye-test-data.sql | 9 ++- 3 files changed, 136 insertions(+), 3 deletions(-) diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js index 42778b6..6b4695d 100644 --- a/d2m/converters/message-to-event.test.js +++ b/d2m/converters/message-to-event.test.js @@ -362,6 +362,41 @@ test("message2event: simple reply to matrix user, reply fallbacks disabled", asy }]) }) +test("message2event: reply with a video", async t => { + const events = await messageToEvent(data.message.reply_with_video, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!kLRqKKUQXcibIMtOpl:cadence.moe", "$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw", { + type: "m.room.message", + content: { + msgtype: "m.text", + body: 'deadpicord "extremity you woke up at 4 am"' + }, + sender: "@_ooye_extremity:cadence.moe" + }) + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.video", + body: "Ins_1960637570.mp4", + filename: "Ins_1960637570.mp4", + url: "mxc://cadence.moe/kMqLycqMURhVpwleWkmASpnU", + external_url: "https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4?ex=65bbee8f&is=65a9798f&hm=ae14f7824c3d526c5e11c162e012e1ee405fd5776e1e9302ed80ccd86503cfda&", + info: { + h: 854, + mimetype: "video/mp4", + size: 860559, + w: 480, + }, + "m.mentions": {}, + "m.relates_to": { + "m.in_reply_to": { + event_id: "$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw" + } + } + }]) +}) + test("message2event: simple reply in thread to a matrix user's reply", async t => { const events = await messageToEvent(data.message.simple_reply_to_reply_in_thread, data.guild.general, {}, { api: { diff --git a/test/data.js b/test/data.js index c2c548d..87ac510 100644 --- a/test/data.js +++ b/test/data.js @@ -1245,6 +1245,101 @@ module.exports = { attachments: [], guild_id: "112760669178241024" }, + reply_with_video: { + id: "1197621094983676007", + type: 19, + content: "", + channel_id: "112760669178241024", + author: { + id: "772659086046658620", + username: "cadence.worm", + avatar: "4b5c4b28051144e4c111f0113a0f1cf1", + discriminator: "0", + public_flags: 0, + premium_type: 2, + flags: 0, + banner: null, + accent_color: null, + global_name: "cadence", + avatar_decoration_data: null, + banner_color: null + }, + attachments: [ + { + id: "1197621094786531358", + filename: "Ins_1960637570.mp4", + size: 860559, + url: "https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4?ex=65bbee8f&is=65a9798f&hm=ae14f7824c3d526c5e11c162e012e1ee405fd5776e1e9302ed80ccd86503cfda&", + proxy_url: "https://media.discordapp.net/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4?ex=65bbee8f&is=65a9798f&hm=ae14f7824c3d526c5e11c162e012e1ee405fd5776e1e9302ed80ccd86503cfda&", + width: 480, + height: 854, + content_type: "video/mp4", + placeholder: "wvcFBABod4gIl3enl6iqfM+s+A==", + placeholder_version: 1 + } + ], + embeds: [], + mentions: [ + { + id: "114147806469554185", + username: "extremity", + avatar: "e0394d500407a8fa93774e1835b8b03a", + discriminator: "0", + public_flags: 768, + premium_type: 2, + flags: 768, + banner: null, + accent_color: null, + global_name: "Extremity", + avatar_decoration_data: null, + banner_color: null + } + ], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2024-01-18T19:18:39.768000+00:00", + edited_timestamp: null, + flags: 0, + components: [], + message_reference: { + channel_id: "112760669178241024", + message_id: "1197612733600895076", + guild_id: "112760669178241024" + }, + referenced_message: { + id: "1197612733600895076", + type: 0, + content: 'deadpicord "extremity you wake up at 4am"', + channel_id: "112760669178241024", + author: { + id: "114147806469554185", + username: "extremity", + avatar: "e0394d500407a8fa93774e1835b8b03a", + discriminator: "0", + public_flags: 768, + premium_type: 2, + flags: 768, + banner: null, + accent_color: null, + global_name: "Extremity", + avatar_decoration_data: null, + banner_color: null + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2024-01-18T18:45:26.259000+00:00", + edited_timestamp: null, + flags: 0, + components: [] + } + }, simple_reply_to_reply_in_thread: { type: 19, tts: false, diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index f99d7da..fac68fd 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -42,7 +42,8 @@ INSERT INTO message_channel (message_id, channel_id) VALUES ('1145688633186193481', '1100319550446252084'), ('1162005526675193909', '1162005314908999790'), ('1162625810109317170', '497161350934560778'), -('1158842413025071135', '176333891320283136'); +('1158842413025071135', '176333891320283136'), +('1197612733600895076', '112760669178241024'); INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES ('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 0, 1), @@ -67,7 +68,8 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part ('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0, 0), ('$0wEdIP8fhTq-P68xwo_gyUw-Zv0KA2aS2tfhdFSrLZc', 'm.room.message', 'm.text', '1162625810109317170', 1, 1, 1), ('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 0, 1), -('$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo', 'm.room.message', 'm.text', '1158842413025071135', 0, 0, 1); +('$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo', 'm.room.message', 'm.text', '1158842413025071135', 0, 0, 1), +('$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw', 'm.room.message', 'm.text', '1197612733600895076', 0, 0, 1); INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'), @@ -84,7 +86,8 @@ INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg', 'mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR'), ('https://cdn.discordapp.com/emojis/1125827250609201255.png', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP'), ('https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024', 'mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX'), -('https://cdn.discordapp.com/emojis/288858540888686602.png', 'mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO'); +('https://cdn.discordapp.com/emojis/288858540888686602.png', 'mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO'), +('https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4', 'mxc://cadence.moe/kMqLycqMURhVpwleWkmASpnU'); INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES ('230201364309868544', 'hippo', 0, 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'), From fbf51dab645bb88e5ceaef671af9ab43cdf913c5 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Jan 2024 12:39:41 +1300 Subject: [PATCH 026/431] Complete code coverage for mxutils --- d2m/event-dispatcher.js | 27 +++++++++--------- m2d/converters/utils.js | 2 +- m2d/converters/utils.test.js | 53 +++++++++++++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 16 deletions(-) diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 6c872fb..3544064 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -26,9 +26,11 @@ const updatePins = sync.require("./actions/update-pins") /** @type {import("../matrix/api")}) */ const api = sync.require("../matrix/api") /** @type {import("../discord/utils")} */ -const utils = sync.require("../discord/utils") +const dUtils = sync.require("../discord/utils") /** @type {import("../discord/discord-command-handler")}) */ const discordCommandHandler = sync.require("../discord/discord-command-handler") +/** @type {import("../m2d/converters/utils")} */ +const mxUtils = require("../m2d/converters/utils") /** @type {any} */ // @ts-ignore bad types from semaphore const Semaphore = require("@chriscdn/promise-semaphore") @@ -68,20 +70,17 @@ module.exports = { stackLines = stackLines.slice(0, cloudstormLine - 2) } } - let formattedBody = "\u26a0 Bridged event from Discord not delivered" - + `
Gateway event: ${gatewayMessage.t}` - + `
${e.toString()}` + + const builder = new mxUtils.MatrixStringBuilder() + builder.addLine("\u26a0 Bridged event from Discord not delivered", "\u26a0 Bridged event from Discord not delivered") + builder.addLine(`Gateway event: ${gatewayMessage.t}`) + builder.addLine(e.toString()) if (stackLines) { - formattedBody += `
Error trace` - + `
${stackLines.join("\n")}
` + builder.addLine(`Error trace:\n${stackLines.join("\n")}`, `
Error trace
${stackLines.join("\n")}
`) } - formattedBody += `
Original payload` - + `
${util.inspect(gatewayMessage.d, false, 4, false)}
`, + builder.addLine("", `
Original payload
${util.inspect(gatewayMessage.d, false, 4, false)}
`) api.sendEvent(roomID, "m.room.message", { - msgtype: "m.text", - body: "\u26a0 Bridged event from Discord not delivered. See formatted content for full details.", - format: "org.matrix.custom.html", - formatted_body: formattedBody, + ...builder.get(), "moe.cadence.ooye.error": { source: "discord", payload: gatewayMessage @@ -113,7 +112,7 @@ module.exports = { const member = guild.members.find(m => m.user?.id === client.user.id) if (!member) return if (!("permission_overwrites" in channel)) continue - const permissions = utils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites) + const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites) const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY if ((permissions & wants) !== wants) continue // We don't have permission to look back in this channel @@ -162,7 +161,7 @@ module.exports = { const lastPin = updatePins.convertTimestamp(channel.last_pin_timestamp) // Permissions check - const permissions = utils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites) + const permissions = dUtils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites) const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY if ((permissions & wants) !== wants) continue // We don't have permission to look up the pins in this channel diff --git a/m2d/converters/utils.js b/m2d/converters/utils.js index b820864..8a83a07 100644 --- a/m2d/converters/utils.js +++ b/m2d/converters/utils.js @@ -72,7 +72,7 @@ class MatrixStringBuilder { /** * @param {string} body - * @param {string} formattedBody + * @param {string} [formattedBody] * @param {any} [condition] */ add(body, formattedBody, condition = true) { diff --git a/m2d/converters/utils.test.js b/m2d/converters/utils.test.js index 9d039fe..76fd824 100644 --- a/m2d/converters/utils.test.js +++ b/m2d/converters/utils.test.js @@ -1,7 +1,10 @@ // @ts-check +const e = new Error("Custom error") + const {test} = require("supertape") -const {eventSenderIsFromDiscord, getEventIDHash} = require("./utils") +const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder} = require("./utils") +const util = require("util") test("sender type: matrix user", t => { t.notOk(eventSenderIsFromDiscord("@cadence:cadence.moe")) @@ -23,3 +26,51 @@ test("event hash: hash is the same each time", t => { test("event hash: hash is different for different inputs", t => { t.notEqual(getEventIDHash("$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe1"), getEventIDHash("$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe2")) }) + +test("MatrixStringBuilder: add, addLine, add same text", t => { + const gatewayMessage = {t: "MY_MESSAGE", d: {display: "Custom message data"}} + let stackLines = e.stack?.split("\n") + + const builder = new MatrixStringBuilder() + builder.addLine("\u26a0 Bridged event from Discord not delivered", "\u26a0 Bridged event from Discord not delivered") + builder.addLine(`Gateway event: ${gatewayMessage.t}`) + builder.addLine(e.toString()) + if (stackLines) { + stackLines = stackLines.slice(0, 2) + stackLines[1] = stackLines[1].replace(/\\/g, "/").replace(/(\s*at ).*(\/m2d\/)/, "$1.$2") + builder.addLine(`Error trace:`, `
Error trace`) + builder.add(`\n${stackLines.join("\n")}`, `
${stackLines.join("\n")}
`) + } + builder.addLine("", `
Original payload
${util.inspect(gatewayMessage.d, false, 4, false)}
`) + + t.deepEqual(builder.get(), { + msgtype: "m.text", + body: "\u26a0 Bridged event from Discord not delivered" + + "\nGateway event: MY_MESSAGE" + + "\nError: Custom error" + + "\nError trace:" + + "\nError: Custom error" + + "\n at ./m2d/converters/utils.test.js:3:11)\n", + format: "org.matrix.custom.html", + formatted_body: "\u26a0 Bridged event from Discord not delivered" + + "
Gateway event: MY_MESSAGE" + + "
Error: Custom error" + + "
Error trace
Error: Custom error\n    at ./m2d/converters/utils.test.js:3:11)
" + + `
Original payload
{ display: 'Custom message data' }
` + }) +}) + +test("MatrixStringBuilder: complete code coverage", t => { + const builder = new MatrixStringBuilder() + builder.add("Line 1") + builder.addParagraph("Line 2") + builder.add("Line 3") + builder.addParagraph("Line 4") + + t.deepEqual(builder.get(), { + msgtype: "m.text", + body: "Line 1\n\nLine 2Line 3\n\nLine 4", + format: "org.matrix.custom.html", + formatted_body: "Line 1

Line 2

Line 3

Line 4

" + }) +}) From 235aee3fef6eaf2f6262dffbfee5338ab5591734 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Jan 2024 12:40:00 +1300 Subject: [PATCH 027/431] Complete code coverage for emoji sprite sheet --- m2d/converters/emoji-sheet.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/m2d/converters/emoji-sheet.js b/m2d/converters/emoji-sheet.js index c05f45d..c85176b 100644 --- a/m2d/converters/emoji-sheet.js +++ b/m2d/converters/emoji-sheet.js @@ -32,6 +32,7 @@ async function compositeMatrixEmojis(mxcs) { // @ts-ignore the signal is slightly different from the type it wants (still works fine) const res = await fetch(url, {agent: false, signal: abortController.signal}) const {stream, mime} = await streamMimeType.getMimeType(res.body) + assert(["image/png", "image/jpeg", "image/webp", "image/gif"].includes(mime), `Mime type ${mime} is impossible for emojis`) if (mime === "image/png" || mime === "image/jpeg" || mime === "image/webp") { /** @type {{info: sharp.OutputInfo, buffer: Buffer}} */ @@ -64,10 +65,6 @@ async function compositeMatrixEmojis(mxcs) { .toBuffer({resolveWithObject: true}) return buffer.data - } else { - // unsupported mime type - console.error(`I don't know what a ${mime} emoji is.`) - return null } } finally { abortController.abort() From aa9c2cc0c7ff7fbe78e4420916e7a2e9c066dcdf Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Jan 2024 12:48:07 +1300 Subject: [PATCH 028/431] Reduce surface for test to break --- m2d/converters/event-to-message.test.js | 5 +---- test/ooye-test-data.sql | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index d5f8455..95139fc 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -543,10 +543,6 @@ test("event2message: lists have appropriate line breaks", async t => { room_id: '!cBxtVRxDlZvSVhJXVK:cadence.moe', sender: '@Milan:tchncs.de', type: 'm.room.message', - }, {}, { - api: { - getStateEvent: async () => ({displayname: "Milan"}) - } }), { ensureJoined: [], @@ -759,6 +755,7 @@ test("event2message: rich reply to a rich reply to a multi-line message should c }, snow: { guild: { + /* c8 ignore next 4 */ searchGuildMembers: (_, options) => { t.fail(`should not search guild members, but actually searched for: ${options.query}`) return [] diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index fac68fd..e9c2fec 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -104,7 +104,8 @@ INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES ('!fGgIymcYWOqjbSRUdV:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), ('!BnKuBPCvyfOkhcUjEu:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), ('!maggESguZBqGBZtSnr:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), -('!CzvdIdUQXgUjDVKxeU:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'); +('!CzvdIdUQXgUjDVKxeU:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), +('!cBxtVRxDlZvSVhJXVK:cadence.moe', '@Milan:tchncs.de', 'Milan', NULL); INSERT INTO lottie (sticker_id, mxc_url) VALUES ('860171525772279849', 'mxc://cadence.moe/ZtvvVbwMIdUZeovWVyGVFCeR'); From f3cacc89fd7fd177225283628a58fffd81f08509 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Jan 2024 16:38:31 +1300 Subject: [PATCH 029/431] m->d: Code coverage for invalid events --- m2d/converters/event-to-message.js | 64 ++++- m2d/converters/event-to-message.test.js | 333 +++++++++++++++++++++++- test/ooye-test-data.sql | 3 +- 3 files changed, 388 insertions(+), 12 deletions(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 31c0255..fdb0418 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -301,8 +301,12 @@ async function handleRoomOrMessageLinks(input, di) { result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, messageID) } else { // 3: Linking to an unknown event that OOYE didn't originally bridge - we can guess messageID from the timestamp - const originalEvent = await di.api.getEvent(roomID, eventID) - if (!originalEvent) continue + let originalEvent + try { + originalEvent = await di.api.getEvent(roomID, eventID) + } catch (e) { + continue // Our homeserver doesn't know about the event, so can't resolve it to a Discord link + } const guessedMessageID = dUtils.timestampToSnowflakeInexact(originalEvent.origin_server_ts) result = MAKE_RESULT.MESSAGE_LINK[resultType](guildID, channelID, guessedMessageID) } @@ -318,7 +322,7 @@ async function handleRoomOrMessageLinks(input, di) { /** * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event * @param {import("discord-api-types/v10").APIGuild} guild - * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer}} di simple-as-nails dependency injection for the matrix API + * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: typeof fetch}} di simple-as-nails dependency injection for the matrix API */ async function eventToMessage(event, guild, di) { /** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */ @@ -393,8 +397,43 @@ async function eventToMessage(event, guild, di) { await (async () => { const repliedToEventId = event.content["m.relates_to"]?.["m.in_reply_to"]?.event_id if (!repliedToEventId) return - let repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId) - if (!repliedToEvent) return + let repliedToEvent + try { + repliedToEvent = await di.api.getEvent(event.room_id, repliedToEventId) + } catch (e) { + // Original event isn't on our homeserver, so we'll *partially* trust the client's reply fallback. + // We'll trust the fallback's quoted content and put it in the reply preview, but we won't trust the authorship info on it. + + // (But if the fallback's quoted content doesn't exist, we give up. There's nothing for us to quote.) + if (event.content["format"] !== "org.matrix.custom.html" || typeof event.content["formatted_body"] !== "string") { + const lines = event.content.body.split("\n") + let stage = 0 + for (let i = 0; i < lines.length; i++) { + if (stage >= 0 && lines[i][0] === ">") stage = 1 + if (stage >= 1 && lines[i].trim() === "") stage = 2 + if (stage === 2 && lines[i].trim() !== "") { + event.content.body = lines.slice(i).join("\n") + break + } + } + return + } + const mxReply = event.content["formatted_body"] + const quoted = mxReply.match(/^
.*?In reply to.*?
(.*)<\/blockquote><\/mx-reply>/)?.[1] + if (!quoted) return + const contentPreviewChunks = chunk( + entities.decodeHTML5Strict( // Remove entities like & " + quoted.replace(/^\s*
.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards + .replace(/(?:\n|
)+/g, " ") // Should all be on one line + .replace(/]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.) + .replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting. + ), 50) + replyLine = "> " + contentPreviewChunks[0] + if (contentPreviewChunks.length > 1) replyLine = replyLine.replace(/[,.']$/, "") + "..." + replyLine += "\n" + return + } + // @ts-ignore const autoEmoji = new Map(select("auto_emoji", ["name", "emoji_id"], {}, "WHERE name = 'L1' OR name = 'L2'").raw().all()) replyLine = `<:L1:${autoEmoji.get("L1")}><:L2:${autoEmoji.get("L2")}>` @@ -408,7 +447,11 @@ async function eventToMessage(event, guild, di) { replyLine += `<@${authorID}>` } else { let senderName = select("member_cache", "displayname", {mxid: repliedToEvent.sender}).pluck().get() - if (!senderName) senderName = sender.match(/@([^:]*)/)?.[1] || sender + if (!senderName) { + const match = sender.match(/@([^:]*)/) + assert(match) + senderName = match[1] + } replyLine += `Ⓜ️**${senderName}**` } // If the event has been edited, the homeserver will include the relation in `unsigned`. @@ -507,7 +550,7 @@ async function eventToMessage(event, guild, di) { if (!match[0].includes("data-mx-emoticon")) break const mxcUrl = match[0].match(/\bsrc="(mxc:\/\/[^"]+)"/) if (mxcUrl) endOfMessageEmojis.unshift(mxcUrl[1]) - if (typeof match.index !== "number") break + assert(typeof match.index === "number", "Your JavaScript implementation does not comply with TC39: https://tc39.es/ecma262/multipage/text-processing.html#sec-regexpbuiltinexec") last = match.index } @@ -563,8 +606,11 @@ async function eventToMessage(event, guild, di) { if (event.content.info?.mimetype?.includes("/")) { mimetype = event.content.info.mimetype } else { - const res = await fetch(url, {method: "HEAD"}) - mimetype = res.headers.get("content-type") || "image/webp" + const res = await di.fetch(url, {method: "HEAD"}) + if (res.status === 200) { + mimetype = res.headers.get("content-type") + } + if (!mimetype) throw new Error(`Server error ${res.status} or missing content-type while detecting sticker mimetype`) } filename += "." + mimetype.split("/")[1] } diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 95139fc..98c153f 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -3,7 +3,7 @@ const {test} = require("supertape") const {eventToMessage} = require("./event-to-message") const data = require("../../test/data") const {MatrixServerError} = require("../../matrix/mreq") -const {db, select} = require("../../passthrough") +const {db, select, discord} = require("../../passthrough") /* c8 ignore next 7 */ function slow() { @@ -855,6 +855,151 @@ test("event2message: rich reply to an already-edited message will quote the new ) }) +test("event2message: rich reply to a missing event will quote from formatted_body without a link", async t => { + let called = 0 + t.deepEqual( + await eventToMessage({ + "type": "m.room.message", + "sender": "@cadence:cadence.moe", + "content": { + "msgtype": "m.text", + "body": "> <@_ooye_kyuugryphon:cadence.moe>\n> > She *sells* *sea*shells by the *sea*shore.\n> But who *sees* the *sea*shells she *sells* sitting sideways?\n\nWhat a tongue-bender...", + "format": "org.matrix.custom.html", + "formatted_body": "
In reply to @_ooye_kyuugryphon:cadence.moe
" + + "
She sells seashells by the seashore.
But who sees the seashells she sells sitting sideways?" + + "
What a tongue-bender...", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup" + } + } + }, + "origin_server_ts": 1693029683016, + "unsigned": { + "age": 91, + "transaction_id": "m1693029682894.510" + }, + "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", + "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" + }, data.guild.general, { + api: { + async getEvent(roomID, eventID) { + called++ + t.equal(roomID, "!fGgIymcYWOqjbSRUdV:cadence.moe") + t.equal(eventID, "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup") + throw new Error("missing event or something") + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "> But who sees the seashells she sells sitting..." + + "\nWhat a tongue-bender...", + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" + }] + } + ) + t.equal(called, 1, "getEvent should be called once") +}) + +test("event2message: rich reply to a missing event without formatted_body will use plaintext body and strip reply fallback", async t => { + let called = 0 + t.deepEqual( + await eventToMessage({ + "type": "m.room.message", + "sender": "@cadence:cadence.moe", + "content": { + "msgtype": "m.text", + "body": "> <@_ooye_kyuugryphon:cadence.moe> Slow news day.\n\nTesting this reply, ignore", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup" + } + } + }, + "origin_server_ts": 1693029683016, + "unsigned": { + "age": 91, + "transaction_id": "m1693029682894.510" + }, + "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", + "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" + }, data.guild.general, { + api: { + async getEvent(roomID, eventID) { + called++ + t.equal(roomID, "!fGgIymcYWOqjbSRUdV:cadence.moe") + t.equal(eventID, "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup") + throw new Error("missing event or something") + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "Testing this reply, ignore", + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" + }] + } + ) + t.equal(called, 1, "getEvent should be called once") +}) + +test("event2message: rich reply to a missing event and no reply fallback will not generate a reply", async t => { + let called = 0 + t.deepEqual( + await eventToMessage({ + "type": "m.room.message", + "sender": "@cadence:cadence.moe", + "content": { + "msgtype": "m.text", + "body": "Testing this reply, ignore.", + "format": "org.matrix.custom.html", + "formatted_body": "Testing this reply, ignore.", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup" + } + } + }, + "origin_server_ts": 1693029683016, + "unsigned": { + "age": 91, + "transaction_id": "m1693029682894.510" + }, + "event_id": "$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8", + "room_id": "!fGgIymcYWOqjbSRUdV:cadence.moe" + }, data.guild.general, { + api: { + async getEvent(roomID, eventID) { + called++ + t.equal(roomID, "!fGgIymcYWOqjbSRUdV:cadence.moe") + t.equal(eventID, "$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cmadeup") + throw new Error("missing event or something") + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "Testing this reply, ignore.", + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" + }] + } + ) + t.equal(called, 1, "getEvent should be called once") +}) + test("event2message: should avoid using blockquote contents as reply preview in rich reply to a sim user", async t => { t.deepEqual( await eventToMessage({ @@ -1822,6 +1967,35 @@ test("event2message: mentioning bridged rooms works", async t => { ) }) +test("event2message: mentioning bridged rooms works (plaintext body)", async t => { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: `I'm just https://matrix.to/#/!BnKuBPCvyfOkhcUjEu:cadence.moe?via=cadence.moe testing channel mentions` + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "I'm just <#1100319550446252084> testing channel mentions", + avatar_url: undefined + }] + } + ) +}) + test("event2message: mentioning known bridged events works (plaintext body)", async t => { t.deepEqual( await eventToMessage({ @@ -1913,7 +2087,7 @@ test("event2message: mentioning known bridged events works (formatted body)", as ) }) -test("event2message: mentioning unknown bridged events works", async t => { +test("event2message: mentioning unknown bridged events can approximate with timestamps", async t => { let called = 0 t.deepEqual( await eventToMessage({ @@ -1957,6 +2131,88 @@ test("event2message: mentioning unknown bridged events works", async t => { t.equal(called, 1, "getEvent should be called once") }) +test("event2message: mentioning events falls back to original link when server doesn't know about it", async t => { + let called = 0 + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: `it was uploaded years ago in amanda-spam` + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOV", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }, {}, { + api: { + async getEvent(roomID, eventID) { + called++ + t.equal(roomID, "!CzvdIdUQXgUjDVKxeU:cadence.moe") + t.equal(eventID, "$zpzx6ABetMl8BrpsFbdZ7AefVU1Y_-t97bJRJM2JyW1") + throw new Error("missing event or something") + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "it was uploaded years ago in [amanda-spam]()", + avatar_url: undefined + }] + } + ) + t.equal(called, 1, "getEvent should be called once") +}) + +test("event2message: mentioning events falls back to original link when the channel-guild isn't in cache", async t => { + t.equal(select("channel_room", "channel_id", {room_id: "!tnedrGVYKFNUdnegvf:tchncs.de"}).pluck().get(), "489237891895768942", "consistency check: this channel-room needs to be in the database for the test to make sense") + t.equal(discord.channels.get("489237891895768942"), undefined, "consistency check: this channel needs to not be in client cache for the test to make sense") + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: `it was uploaded years ago in ex-room-doesnt-exist-any-more` + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOX", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }, {}, { + api: { + /* c8 skip next 3 */ + async getEvent() { + t.fail("getEvent should not be called because it should quit early due to no channel-guild") + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "it was uploaded years ago in [ex-room-doesnt-exist-any-more]()", + avatar_url: undefined + }] + } + ) +}) + test("event2message: link to event in an unknown room", async t => { t.deepEqual( await eventToMessage({ @@ -2382,6 +2638,79 @@ test("event2message: stickers work", async t => { ) }) +test("event2message: stickers fetch mimetype from server when mimetype not provided", async t => { + let called = 0 + t.deepEqual( + await eventToMessage({ + type: "m.sticker", + sender: "@cadence:cadence.moe", + content: { + body: "YESYESYES", + url: "mxc://cadence.moe/ybOWQCaXysnyUGuUCaQlTGJf" + }, + event_id: "$mL-eEVWCwOvFtoOiivDP7gepvf-fTYH6_ioK82bWDI0", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }, {}, { + async fetch(url, options) { + called++ + t.equal(url, "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/ybOWQCaXysnyUGuUCaQlTGJf") + t.equal(options.method, "HEAD") + return { + status: 200, + headers: new Map([ + ["content-type", "image/gif"] + ]) + } + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "", + avatar_url: undefined, + attachments: [{id: "0", filename: "YESYESYES.gif"}], + pendingFiles: [{name: "YESYESYES.gif", url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/ybOWQCaXysnyUGuUCaQlTGJf"}] + }] + } + ) + t.equal(called, 1, "sticker headers should be fetched") +}) + +test("event2message: stickers with unknown mimetype are not allowed", async t => { + let called = 0 + try { + await eventToMessage({ + type: "m.sticker", + sender: "@cadence:cadence.moe", + content: { + body: "something", + url: "mxc://cadence.moe/ybOWQCaXysnyUGuUCaQlTGJe" + }, + event_id: "$mL-eEVWCwOvFtoOiivDP7gepvf-fTYH6_ioK82bWDI0", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }, {}, { + async fetch(url, options) { + called++ + t.equal(url, "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/ybOWQCaXysnyUGuUCaQlTGJe") + t.equal(options.method, "HEAD") + return { + status: 404, + headers: new Map([ + ["content-type", "application/json"] + ]) + } + } + }) + /* c8 ignore next */ + t.fail("should throw an error") + } catch (e) { + t.match(e.toString(), "mimetype") + } +}) + test("event2message: static emojis work", async t => { t.deepEqual( await eventToMessage({ diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index e9c2fec..8da0128 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -11,7 +11,8 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom ('1162005314908999790', '!FuDZhlOAtqswlyxzeR:cadence.moe', 'Hey.', NULL, '1100319550446252084', NULL), ('297272183716052993', '!rEOspnYqdOalaIFniV:cadence.moe', 'general', NULL, NULL, NULL), ('122155380120748034', '!cqeGDbPiMFAhLsqqqq:cadence.moe', 'cadences-mind', 'coding', NULL, NULL), -('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS'); +('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS'), +('489237891895768942', '!tnedrGVYKFNUdnegvf:tchncs.de', 'ex-room-doesnt-exist-any-more', NULL, NULL, NULL); INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES ('0', 'bot', '_ooye_bot', '@_ooye_bot:cadence.moe'), From c3dc3c89b57af6d312af6b971dce735f937cf6c7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Jan 2024 16:42:00 +1300 Subject: [PATCH 030/431] Code coverage for migrate.js --- db/migrate.js | 1 + db/migrations/.baby | 0 2 files changed, 1 insertion(+) create mode 100644 db/migrations/.baby diff --git a/db/migrate.js b/db/migrate.js index 7c1faf9..57b5cbf 100644 --- a/db/migrate.js +++ b/db/migrate.js @@ -16,6 +16,7 @@ async function migrate(db) { let migrationRan = false for (const filename of files) { + /* c8 ignore next - we can't unit test this, but it's run on every real world bridge startup */ if (progress >= filename) continue console.log(`Applying database migration ${filename}`) if (filename.endsWith(".sql")) { diff --git a/db/migrations/.baby b/db/migrations/.baby new file mode 100644 index 0000000..e69de29 From 0ab81d3d88316d1f458d66727fbcf397c1f3bf86 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Jan 2024 16:43:12 +1300 Subject: [PATCH 031/431] Put expressions with the actions, where it belongs --- d2m/actions/create-space.js | 4 ++-- d2m/{converters => actions}/expression.js | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) rename d2m/{converters => actions}/expression.js (96%) diff --git a/d2m/actions/create-space.js b/d2m/actions/create-space.js index 4e30bcd..7d50199 100644 --- a/d2m/actions/create-space.js +++ b/d2m/actions/create-space.js @@ -13,8 +13,8 @@ const api = sync.require("../../matrix/api") const file = sync.require("../../matrix/file") /** @type {import("./create-room")} */ const createRoom = sync.require("./create-room") -/** @type {import("../converters/expression")} */ -const expression = sync.require("../converters/expression") +/** @type {import("./expression")} */ +const expression = sync.require("./expression") /** @type {import("../../matrix/kstate")} */ const ks = sync.require("../../matrix/kstate") diff --git a/d2m/converters/expression.js b/d2m/actions/expression.js similarity index 96% rename from d2m/converters/expression.js rename to d2m/actions/expression.js index 1c52c98..b7b5d5a 100644 --- a/d2m/converters/expression.js +++ b/d2m/actions/expression.js @@ -1,10 +1,9 @@ // @ts-check -const assert = require("assert").strict const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../../passthrough") -const {discord, sync, db, select} = passthrough +const {sync, db} = passthrough /** @type {import("../../matrix/file")} */ const file = sync.require("../../matrix/file") From 5ef5dbb2e8d305bf75f6948f3024dfdb0e133ca5 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 19 Jan 2024 16:43:30 +1300 Subject: [PATCH 032/431] Write more "add a new event type" documentation --- docs/how-to-add-a-new-event-type.md | 71 ++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/docs/how-to-add-a-new-event-type.md b/docs/how-to-add-a-new-event-type.md index ea808e0..ec9cd7b 100644 --- a/docs/how-to-add-a-new-event-type.md +++ b/docs/how-to-add-a-new-event-type.md @@ -32,13 +32,13 @@ What does it look like on Discord-side? This is an API request to get the pinned messages. To update this, an API request will pin or unpin any specific message, adding or removing it from the list. -## What will the converter look like? +## What will the converter do? The converter will be very different in both directions. -For d2m, we will get the list of pinned messages, we will convert each message ID into the ID of an event we already have, and then we will set the entire `m.room.pinned_events` state to that list. +**For d2m, we will get the list of pinned messages, we will convert each message ID into the ID of an event we already have, and then we will set the entire `m.room.pinned_events` state to that list.** -For m2d, we will have to diff the list of pinned messages against the previous version of the list, and for each event that was pinned or unpinned, we will send an API request to Discord to change its state. +**For m2d, we will have to diff the list of pinned messages against the previous version of the list, and for each event that was pinned or unpinned, we will send an API request to Discord to change its st**ate. ## Missing messages @@ -53,7 +53,7 @@ In this situation we need to stop and think about the possible paths forward we The latter method would still make the message appear at the bottom of the timeline for most Matrix clients, since for most the timestamp doesn't determine the actual _order._ It would then be confusing why an odd message suddenly appeared, because a pins change isn't that noticable in the room. -To avoid this problem, I'll just go with the former method and ignore the message, so Matrix will only have some of the pins that Discord has. We will need to watch out if a Matrix user edits this list of partial pins, because if we _only_ pinned things on Discord that were pinned on Matrix, those partial pins Discord would be lost from Discord side. +To avoid this problem, I'll just go with the former method and ignore the message, so Matrix will only have some of the pins that Discord has. We will need to watch out if a Matrix user edits this list of partial pins, because if we _only_ pinned things on Discord that were pinned on Matrix, then pins Matrix doesn't know about would be lost from Discord side. In this situation I will prefer to keep the pins list inconsistent between both sides and only bridge _changes_ to the list. @@ -61,7 +61,9 @@ If you were implementing this for real, you might have made different decisions ## Test data for the d2m converter -Let's start writing the d2m converter. It's helpful to write unit tests for Out Of Your Element, since this lets you check if it worked without having to start up a local copy of the bridge or play around with the interface. +Let's start writing the d2m converter. It's helpful to write automated tests for Out Of Your Element, since this lets you check if it worked without having to start up a local copy of the bridge or mess around with the interface. + +To test the Discord-to-Matrix pin converter, we'll need some samples of Discord message objects. Then we can put these sample message objects through the converter and check what comes out the other side. Normally for getting test data, I would `curl` the Discord API to grab some real data and put it into `data.js` (and possibly also `ooye-test-data.sql`. But this time, I'll fabricate some test data. Here it is: @@ -74,7 +76,7 @@ Normally for getting test data, I would `curl` the Discord API to grab some real ] ``` -"These aren't message objects!" I hear you cry. Correct. I already know that my implementation is not going to care about any properties on these message object other than the IDs, so I'm just making a list of IDs to save time. +"These aren't message objects!" I hear you cry. Correct. I already know that my implementation is not going to care about any properties on these message object other than the IDs, so to save time, I'm just making a list of IDs. These IDs were carefully chosen. The first three are already in `ooye-test-data.sql` and are associated with event IDs. This is great, because in our test case, the Discord IDs will be converted to those event IDs. The fourth ID doesn't exist on Matrix-side. This is to test that partial pins are handled as expected, like I wrote in the previous section. @@ -104,7 +106,7 @@ index c36f252..4919beb 100644 ## Writing the d2m converter -We can write a function that operates on this data to convert it to events. This is a _converter,_ not an _action._ it won't _do_ anything by itself. So it goes in the converters folder. The actual function is pretty simple since I've already planned what to do: +We can write a function that operates on this data to convert it to events. This is a _converter,_ not an _action._ It won't _do_ anything by itself. So it goes in the converters folder. I've already planned (in the "What will the converter do?" section) what to do, so writing the function is pretty simple: ```diff diff --git a/d2m/converters/pins-to-list.js b/d2m/converters/pins-to-list.js @@ -133,9 +135,36 @@ index 0000000..e4107be +module.exports.pinsToList = pinsToList ``` +### Explaining the code + +All converters have a `function` which does the work, and the function is added to `module.exports` so that other files can use it. + +Importing `select` from `passthrough` lets us do database access. Calling the `select` function can select from OOYE's own SQLite database. If you want to see what's in the database, look at `ooye-test-data.sql` for test data, or open `ooye.db` for real data from your own bridge. + +The comments `// @ts-check`, `/** @type ... */`, and `/** @param ... */` provide type-based autosuggestions when editing in Visual Studio Code. + +Here's the code I haven't yet discussed: + +```js +function pinsToList(pins) { + const result = [] + for (const message of pins) { + const eventID = select("event_message", "event_id", {message_id: message.id}).pluck().get() + if (eventID) result.push(eventID) + } + return result +} +``` + +It will go through each `message` in `pins`. For each message, it will look up the corresponding Matrix event in the database, and if found, it will add it to `result`. + +The `select` line will run this SQL: `SELECT event_id FROM event_message WHERE message_id = {the message ID}` and will return the event ID as a string or null. + +For any database experts worried about an SQL query inside a loop, the N+1 problem does not apply to SQLite because the queries are executed in the same process rather than crossing a process (and network) boundary. https://www.sqlite.org/np1queryprob.html + ## Test case for the d2m converter -There's not much room for bugs in this function. A single manual test that it works would be good enough for me. But since this is an example of how you can add your own, let's add a test case for this. We'll take the data we just prepared and process it through the function we just wrote: +There's not much room for bugs in this function. A single manual test that it works would be good enough for me. But since this is an example of how you can add your own, let's add a test case for this. The testing code will take the data we just prepared and process it through the `pinsToList` function we just wrote. Then, it will check the result is what we expected. ```diff diff --git a/d2m/converters/pins-to-list.test.js b/d2m/converters/pins-to-list.test.js @@ -177,6 +206,18 @@ index 5cc851e..280503d 100644 Good to go. +### Explaining the code + +`require("supertape")` is a library that helps with testing and printing test results. `data = require("../../test/data")` is the file we edited earlier in the "Test data for the d2m converter" section. `require("./pins-to-list")` is the function we want to test. + +Here is how you declare a test: `test("pins2list: converts known IDs, ignores unknown IDs", t => {` The string describes what you are trying to test and it will be displayed if the test fails. + +`result = pinsToList(data.pins.faked)` is calling the implementation function we wrote. + +`t.deepEqual(actual, expected)` will check whether the `actual` result value is the same as our `expected` result value. If it's not, it'll mark that as a failed test. + +### Run the test! + ``` ><> $ npm t @@ -209,7 +250,11 @@ Oh no! (I promise I didn't make it fail for demonstration purposes, this was act ('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', 1, 1), ``` -Explanation: This Discord message `1141501302736695316` is actually part of 2 different Matrix events, `$mtR...` and `$51f...`. This often happens when a Discord user uploads an image with a caption. Matrix doesn't support combined image+text events, so the image and the text have to be bridged to separate events. We should consider the text to be the primary part, and pin that, and consider the image to be the secondary part, and not pin that. +Explanation: This Discord message `1141501302736695316` is actually part of 2 different Matrix events, `$mtR...` and `$51f...`. This often happens when a Discord user uploads an image with a caption. Matrix doesn't support combined image+text events, so the image and the text have to be bridged to separate events. + +In the current code, `pinsToList` is picking ALL the associated event IDs, and then `.get` is forcing it to limit that list to 1. It doesn't care which, so it's essentially random which event it wants to pin. + +We should make a decision on which event is more important. You can make whatever decision you want - you could even make it pin every event associated with a message - but I've decided that the text should be the primary part and be pinned, and the image should be considered a secondary part and left unpinned. We already have a column `part` in the `event_message` table for this reason! When `part = 0`, that's the primary part. I'll edit the converter to actually use that column: @@ -229,6 +274,8 @@ index e4107be..f401de2 100644 return result ``` +As long as the database is consistent, this new `select` will return at most 1 event, always choosing the primary part. + ``` ><> $ npm t @@ -341,6 +388,8 @@ I try to keep as much logic as possible out of the actions and in the converters ## See if it works +Since the automated tests pass, let's start up the bridge and run our nice new code: + ``` node start.js ``` @@ -359,7 +408,7 @@ I expected that to be the end of the guide, but after some time, I noticed a new [After some investigation,](https://gitdab.com/cadence/out-of-your-element/issues/16) it turns out Discord puts the most recently pinned message at the start of the array and displays the array in forwards order, while Matrix puts the most recently pinned message at the end of the array and displays the array in reverse order. -We'll fix this by reversing the order of the list of pins before we store it. I'll do this in the converter. +We can fix this by reversing the order of the list of pins before we store it. The converter can do this: ```diff diff --git a/d2m/converters/pins-to-list.js b/d2m/converters/pins-to-list.js @@ -405,7 +454,7 @@ index c2e3774..92e5678 100644 Pass! ``` -Next time a message is pinned or unpinned on Discord, the order should be updated on Matrix. +Next time a message is pinned or unpinned on Discord, OOYE should update the order of all the pins on Matrix. ## Notes on missed events From e49dc18e67b081e88fcb04f0b9a459cdec1218ab Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 01:01:34 +1300 Subject: [PATCH 033/431] Implement the speedbump --- d2m/actions/delete-message.js | 10 +++-- d2m/actions/speedbump.js | 51 +++++++++++++++++++++++++ d2m/discord-client.js | 11 +++++- d2m/event-dispatcher.js | 12 ++++-- db/migrations/0009-add-speedbump-id.sql | 6 +++ db/orm-defs.d.ts | 2 + docs/pluralkit-notetaking.md | 4 +- m2d/converters/event-to-message.test.js | 6 +-- stdin.js | 1 + 9 files changed, 90 insertions(+), 13 deletions(-) create mode 100644 d2m/actions/speedbump.js create mode 100644 db/migrations/0009-add-speedbump-id.sql diff --git a/d2m/actions/delete-message.js b/d2m/actions/delete-message.js index cca5d25..496d827 100644 --- a/d2m/actions/delete-message.js +++ b/d2m/actions/delete-message.js @@ -4,21 +4,25 @@ const passthrough = require("../../passthrough") const {sync, db, select, from} = passthrough /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") +/** @type {import("./speedbump")} */ +const speedbump = sync.require("./speedbump") /** * @param {import("discord-api-types/v10").GatewayMessageDeleteDispatchData} data */ async function deleteMessage(data) { - const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get() - if (!roomID) return + const row = select("channel_room", ["room_id", "speedbump_id", "speedbump_checked"], {channel_id: data.channel_id}).get() + if (!row) return const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all() db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(data.id) db.prepare("DELETE FROM event_message WHERE message_id = ?").run(data.id) for (const eventID of eventsToRedact) { // Unfortunately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs - await api.redactEvent(roomID, eventID) + await api.redactEvent(row.room_id, eventID) } + + speedbump.updateCache(data.channel_id, row.speedbump_id, row.speedbump_checked) } /** diff --git a/d2m/actions/speedbump.js b/d2m/actions/speedbump.js new file mode 100644 index 0000000..b56cbf6 --- /dev/null +++ b/d2m/actions/speedbump.js @@ -0,0 +1,51 @@ +// @ts-check + +const DiscordTypes = require("discord-api-types/v10") +const passthrough = require("../../passthrough") +const {discord, db} = passthrough + +const SPEEDBUMP_SPEED = 4000 // 4 seconds delay +const SPEEDBUMP_UPDATE_FREQUENCY = 2 * 60 * 60 // 2 hours + +/** @type {Set} */ +const KNOWN_BOTS = new Set([ + "466378653216014359" // PluralKit +]) + +/** + * Fetch new speedbump data for the channel and put it in the database as cache + * @param {string} channelID + * @param {string?} speedbumpID + * @param {number?} speedbumpChecked + */ +async function updateCache(channelID, speedbumpID, speedbumpChecked) { + const now = Math.floor(Date.now() / 1000) + if (speedbumpChecked && now - speedbumpChecked < SPEEDBUMP_UPDATE_FREQUENCY) return + const webhooks = await discord.snow.webhook.getChannelWebhooks(channelID) + const found = webhooks.find(b => KNOWN_BOTS.has(b.application_id))?.application_id || null + db.prepare("UPDATE channel_room SET speedbump_id = ?, speedbump_checked = ? WHERE channel_id = ?").run(found, now, channelID) +} + +/** @type {Set} set of messageID */ +const bumping = new Set() + +/** + * Slow down a message. After it passes the speedbump, return whether it's okay or if it's been deleted. + * @param {string} messageID + */ +async function doSpeedbump(messageID) { + bumping.add(messageID) + await new Promise(resolve => setTimeout(resolve, SPEEDBUMP_SPEED)) + return !bumping.delete(messageID) +} + +/** + * @param {string} messageID + */ +function onMessageDelete(messageID) { + bumping.delete(messageID) +} + +module.exports.updateCache = updateCache +module.exports.doSpeedbump = doSpeedbump +module.exports.onMessageDelete = onMessageDelete diff --git a/d2m/discord-client.js b/d2m/discord-client.js index b1a1e81..80dcbcf 100644 --- a/d2m/discord-client.js +++ b/d2m/discord-client.js @@ -47,7 +47,16 @@ class DiscordClient { if (listen !== "no") { this.cloud.on("event", message => discordPackets.onPacket(this, message, listen)) } - this.cloud.on("error", console.error) + + const addEventLogger = (eventName, logName) => { + this.cloud.on(eventName, (...args) => { + const d = new Date().toISOString().slice(0, 19) + console.error(`[${d} Client ${logName}]`, ...args) + }) + } + addEventLogger("error", "Error") + addEventLogger("disconnected", "Disconnected") + addEventLogger("ready", "Ready") } } diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 3544064..3a80d0a 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -31,6 +31,8 @@ const dUtils = sync.require("../discord/utils") const discordCommandHandler = sync.require("../discord/discord-command-handler") /** @type {import("../m2d/converters/utils")} */ const mxUtils = require("../m2d/converters/utils") +/** @type {import("./actions/speedbump")} */ +const speedbump = sync.require("./actions/speedbump") /** @type {any} */ // @ts-ignore bad types from semaphore const Semaphore = require("@chriscdn/promise-semaphore") @@ -236,9 +238,12 @@ module.exports = { if (message.author.username === "Deleted User") return // Nothing we can do for deleted users. if (message.webhook_id) { const row = select("webhook", "webhook_id", {webhook_id: message.webhook_id}).pluck().get() - if (row) { - // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. - return + if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. + } else { + const speedbumpID = select("channel_room", "speedbump_id", {channel_id: message.channel_id}).pluck().get() + if (speedbumpID) { + const affected = await speedbump.doSpeedbump(message.id) + if (affected) return } } const channel = client.channels.get(message.channel_id) @@ -299,6 +304,7 @@ module.exports = { * @param {DiscordTypes.GatewayMessageDeleteDispatchData} data */ async onMessageDelete(client, data) { + speedbump.onMessageDelete(data.id) await deleteMessage.deleteMessage(data) }, diff --git a/db/migrations/0009-add-speedbump-id.sql b/db/migrations/0009-add-speedbump-id.sql new file mode 100644 index 0000000..e971146 --- /dev/null +++ b/db/migrations/0009-add-speedbump-id.sql @@ -0,0 +1,6 @@ +BEGIN TRANSACTION; + +ALTER TABLE channel_room ADD COLUMN speedbump_id TEXT; +ALTER TABLE channel_room ADD COLUMN speedbump_checked INTEGER; + +COMMIT; diff --git a/db/orm-defs.d.ts b/db/orm-defs.d.ts index 64e5c77..f0f9a67 100644 --- a/db/orm-defs.d.ts +++ b/db/orm-defs.d.ts @@ -7,6 +7,8 @@ export type Models = { thread_parent: string | null custom_avatar: string | null last_bridged_pin_timestamp: number | null + speedbump_id: string | null + speedbump_checked: number | null } event_message: { diff --git a/docs/pluralkit-notetaking.md b/docs/pluralkit-notetaking.md index 697cdee..d03ddb7 100644 --- a/docs/pluralkit-notetaking.md +++ b/docs/pluralkit-notetaking.md @@ -85,7 +85,9 @@ OOYE's speedbump will prevent the edit command appearing at all on Matrix-side, ## Database schema -TBD +* channel_room + + speedbump_id - the ID of the webhook that may be proxying in this channel + + speedbump_checked - time in unix seconds when the webhooks were last queried ## Unsolved problems diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 98c153f..4f1c1dd 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -581,10 +581,6 @@ test("event2message: ordered list start attribute works", async t => { room_id: '!cBxtVRxDlZvSVhJXVK:cadence.moe', sender: '@Milan:tchncs.de', type: 'm.room.message', - }, {}, { - api: { - getStateEvent: async () => ({displayname: "Milan"}) - } }), { ensureJoined: [], @@ -2194,7 +2190,7 @@ test("event2message: mentioning events falls back to original link when the chan } }, {}, { api: { - /* c8 skip next 3 */ + /* c8 ignore next 3 */ async getEvent() { t.fail("getEvent should not be called because it should quit early due to no channel-guild") } diff --git a/stdin.js b/stdin.js index da69d7c..5e23f72 100644 --- a/stdin.js +++ b/stdin.js @@ -17,6 +17,7 @@ const file = sync.require("./matrix/file") const sendEvent = sync.require("./m2d/actions/send-event") const eventDispatcher = sync.require("./d2m/event-dispatcher") const updatePins = sync.require("./d2m/actions/update-pins") +const speedbump = sync.require("./d2m/actions/speedbump") const ks = sync.require("./matrix/kstate") const guildID = "112760669178241024" From 845f93e5d05b67af1c28bf8acddb3d8b278f7079 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 17:03:03 +1300 Subject: [PATCH 034/431] Work around the Content-Length issue --- matrix/mreq.js | 15 +++++++++++++-- package-lock.json | 12 ++++++++++++ package.json | 1 + readme.md | 1 + registration.example.yaml | 1 + scripts/seed.js | 12 +++++++++++- types.d.ts | 1 + 7 files changed, 40 insertions(+), 3 deletions(-) diff --git a/matrix/mreq.js b/matrix/mreq.js index 0bd7505..70d9ac7 100644 --- a/matrix/mreq.js +++ b/matrix/mreq.js @@ -2,6 +2,8 @@ const fetch = require("node-fetch").default const mixin = require("mixin-deep") +const stream = require("stream") +const getStream = require("get-stream") const passthrough = require("../passthrough") const { sync } = passthrough @@ -27,9 +29,15 @@ class MatrixServerError extends Error { * @param {any} [extra] */ async function mreq(method, url, body, extra = {}) { + if (body == undefined || Object.is(body.constructor, Object)) { + body = JSON.stringify(body) + } else if (body instanceof stream.Readable && reg.ooye.content_length_workaround) { + body = await getStream.buffer(body) + } + const opts = mixin({ method, - body: (body == undefined || Object.is(body.constructor, Object)) ? JSON.stringify(body) : body, + body, headers: { Authorization: `Bearer ${reg.as_token}` } @@ -39,7 +47,10 @@ async function mreq(method, url, body, extra = {}) { const res = await fetch(baseUrl + url, opts) const root = await res.json() - if (!res.ok || root.errcode) throw new MatrixServerError(root, {baseUrl, url, ...opts}) + if (!res.ok || root.errcode) { + delete opts.headers.Authorization + throw new MatrixServerError(root, {baseUrl, url, ...opts}) + } return root } diff --git a/package-lock.json b/package-lock.json index 054e701..4fb1269 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "deep-equal": "^2.2.3", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", + "get-stream": "^6.0.1", "giframe": "github:cloudrac3r/giframe#v0.4.1", "heatsync": "^2.4.1", "html-template-tag": "github:cloudrac3r/html-template-tag#v5.0", @@ -1456,6 +1457,17 @@ "source-map": "^0.6.1" } }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/giframe": { "version": "0.3.0", "resolved": "git+ssh://git@github.com/cloudrac3r/giframe.git#1630f4d3b2bf5acd197409c85edd11e0da72d0a1", diff --git a/package.json b/package.json index 58a8674..4aba845 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "deep-equal": "^2.2.3", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", + "get-stream": "^6.0.1", "giframe": "github:cloudrac3r/giframe#v0.4.1", "heatsync": "^2.4.1", "html-template-tag": "github:cloudrac3r/html-template-tag#v5.0", diff --git a/readme.md b/readme.md index e7baf18..df42234 100644 --- a/readme.md +++ b/readme.md @@ -164,6 +164,7 @@ Follow these steps: * (8) snowtransfer: Discord API library with bring-your-own-caching that I trust. * (0) deep-equal: It's already pulled in by supertape. * (1) discord-markdown: This is my fork! +* (0) get-stream: Only needed if content_length_workaround is true. * (0) giframe: This is my fork! * (1) heatsync: Module hot-reloader that I trust. * (0) entities: Looks fine. No dependencies. diff --git a/registration.example.yaml b/registration.example.yaml index 9e7cd2c..9a562cd 100644 --- a/registration.example.yaml +++ b/registration.example.yaml @@ -18,6 +18,7 @@ ooye: max_file_size: 5000000 server_name: [the part after the colon in your matrix id, like cadence.moe] server_origin: [the full protocol and domain of your actual matrix server's location, with no trailing slash, like https://matrix.cadence.moe] + content_length_workaround: false invite: # uncomment this to auto-invite the named user to newly created spaces and mark them as admin (PL 100) everywhere # - @cadence:cadence.moe diff --git a/scripts/seed.js b/scripts/seed.js index 4dc08e0..20cb5dd 100644 --- a/scripts/seed.js +++ b/scripts/seed.js @@ -71,7 +71,17 @@ async function uploadAutoEmoji(guild, name, filename) { try { await api.register(reg.sender_localpart) } catch (e) { - if (e.errcode === "M_USER_IN_USE" || e.data?.error !== "Internal server error") throw e // "Internal server error" is the only OK error because Synapse says this if you try to register the same username twice. + if (e.errcode === "M_USER_IN_USE" || e.data?.error === "Internal server error") { + // "Internal server error" is the only OK error because older versions of Synapse say this if you try to register the same username twice. + } else if (e.data?.error?.includes("Content-Length")) { + die(`OOYE cannot stream uploads to Synapse. Please choose one of these workarounds:` + + `\n * Run an nginx reverse proxy to Synapse, and point registration.yaml's` + + `\n \`server_origin\` to nginx` + + `\n * Set \`content_length_workaround: true\` in registration.yaml (this will` + + `\n halve the speed of bridging d->m files)`) + } else { + throw e + } } // upload initial images... diff --git a/types.d.ts b/types.d.ts index 9e9d72b..477b41d 100644 --- a/types.d.ts +++ b/types.d.ts @@ -21,6 +21,7 @@ export type AppServiceRegistrationConfig = { max_file_size: number server_name: string server_origin: string + content_length_workaround: boolean invite: string[] } old_bridge?: { From 706b37669b59835f678d431bc28372f40005c344 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 17:25:58 +1300 Subject: [PATCH 035/431] Move Content-Length detection logic --- matrix/mreq.js | 8 ++++++++ scripts/seed.js | 6 ------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/matrix/mreq.js b/matrix/mreq.js index 70d9ac7..d515846 100644 --- a/matrix/mreq.js +++ b/matrix/mreq.js @@ -48,6 +48,14 @@ async function mreq(method, url, body, extra = {}) { const root = await res.json() if (!res.ok || root.errcode) { + if (root.error?.includes("Content-Length")) { + console.error(`OOYE cannot stream uploads to Synapse. Please choose one of these workarounds:` + + `\n * Run an nginx reverse proxy to Synapse, and point registration.yaml's` + + `\n \`server_origin\` to nginx` + + `\n * Set \`content_length_workaround: true\` in registration.yaml (this will` + + `\n halve the speed of bridging d->m files)`) + throw new Error("Synapse is not accepting stream uploads, see the message above.") + } delete opts.headers.Authorization throw new MatrixServerError(root, {baseUrl, url, ...opts}) } diff --git a/scripts/seed.js b/scripts/seed.js index 20cb5dd..1a21490 100644 --- a/scripts/seed.js +++ b/scripts/seed.js @@ -73,12 +73,6 @@ async function uploadAutoEmoji(guild, name, filename) { } catch (e) { if (e.errcode === "M_USER_IN_USE" || e.data?.error === "Internal server error") { // "Internal server error" is the only OK error because older versions of Synapse say this if you try to register the same username twice. - } else if (e.data?.error?.includes("Content-Length")) { - die(`OOYE cannot stream uploads to Synapse. Please choose one of these workarounds:` - + `\n * Run an nginx reverse proxy to Synapse, and point registration.yaml's` - + `\n \`server_origin\` to nginx` - + `\n * Set \`content_length_workaround: true\` in registration.yaml (this will` - + `\n halve the speed of bridging d->m files)`) } else { throw e } From 67dc31f747140c8ea64e5fc796d1bc6674270358 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 18:23:48 +1300 Subject: [PATCH 036/431] Fix tests that hard-coded cadence.moe --- test/test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test.js b/test/test.js index 4663b18..a57231d 100644 --- a/test/test.js +++ b/test/test.js @@ -14,6 +14,7 @@ const db = new sqlite(":memory:") const reg = require("../matrix/read-registration") reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded +reg.ooye.server_name = "cadence.moe" reg.ooye.invite = ["@test_auto_invite:example.org"] const sync = new HeatSync({watchFS: false}) From 8591ea5c1f67a5fa6ba615f9f731d3605171355c Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 23:13:39 +1300 Subject: [PATCH 037/431] Explain how to get in the rooms --- readme.md | 4 ++++ registration.example.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index df42234..05d8f9c 100644 --- a/readme.md +++ b/readme.md @@ -96,6 +96,10 @@ Follow these steps: * $ `npm run addbot` * $ `./addbot.sh` +Now any message on Discord will create the corresponding rooms on Matrix-side. After the rooms have been created, Matrix and Discord users can chat back and forth. + +To get into the rooms on your Matrix account, either add yourself to `invite` in `registration.yaml`, or use the `//invite [your mxid here]` command on Discord. + # Development setup * Be sure to install dependencies with `--save-dev` so you can run the tests. diff --git a/registration.example.yaml b/registration.example.yaml index 9a562cd..dd217db 100644 --- a/registration.example.yaml +++ b/registration.example.yaml @@ -21,4 +21,4 @@ ooye: content_length_workaround: false invite: # uncomment this to auto-invite the named user to newly created spaces and mark them as admin (PL 100) everywhere - # - @cadence:cadence.moe + # - '@cadence:cadence.moe' From 0e75c23aee4aa11787470a2ab2c6faaba12abc94 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 23:45:47 +1300 Subject: [PATCH 038/431] Have to join user before announcing thread --- d2m/actions/announce-thread.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/d2m/actions/announce-thread.js b/d2m/actions/announce-thread.js index 86c6412..324c7a5 100644 --- a/d2m/actions/announce-thread.js +++ b/d2m/actions/announce-thread.js @@ -8,6 +8,8 @@ const {discord, sync, db, select} = passthrough const threadToAnnouncement = sync.require("../converters/thread-to-announcement") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") +/** @type {import("./register-user")} */ +const registerUser = sync.require("./register-user") /** * @param {string} parentRoomID @@ -15,10 +17,10 @@ const api = sync.require("../../matrix/api") * @param {import("discord-api-types/v10").APIThreadChannel} thread */ async function announceThread(parentRoomID, threadRoomID, thread) { - const creatorMxid = select("sim", "mxid", {user_id: thread.owner_id}).pluck().get() - - const content = await threadToAnnouncement.threadToAnnouncement(parentRoomID, threadRoomID, creatorMxid, thread, {api}) - + assert(thread.owner_id) + // @ts-ignore + const creatorMxid = await registerUser.ensureSimJoined({id: thread.owner_id}, parentRoomID) + const content = await threadToAnnouncement.threadToAnnouncement(parentRoomID, threadRoomID, creatorMxid, thread, {api}) await api.sendEvent(parentRoomID, "m.room.message", content, creatorMxid) } From 988cb9408d08f70e9bae0a94fd3304ac14797849 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 20 Jan 2024 23:51:26 +1300 Subject: [PATCH 039/431] Fix DI on eventToMessage --- m2d/actions/send-event.js | 2 +- m2d/converters/event-to-message.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/m2d/actions/send-event.js b/m2d/actions/send-event.js index 6b7d3b8..4849740 100644 --- a/m2d/actions/send-event.js +++ b/m2d/actions/send-event.js @@ -75,7 +75,7 @@ async function sendEvent(event) { // no need to sync the matrix member to the other side. but if I did need to, this is where I'd do it - let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow}) + let {messagesToEdit, messagesToSend, messagesToDelete, ensureJoined} = await eventToMessage.eventToMessage(event, guild, {api, snow: discord.snow, fetch}) messagesToEdit = await Promise.all(messagesToEdit.map(async e => { e.message = await resolvePendingFiles(e.message) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index fdb0418..878dcf5 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -322,7 +322,7 @@ async function handleRoomOrMessageLinks(input, di) { /** * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event * @param {import("discord-api-types/v10").APIGuild} guild - * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: typeof fetch}} di simple-as-nails dependency injection for the matrix API + * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"]}} di simple-as-nails dependency injection for the matrix API */ async function eventToMessage(event, guild, di) { /** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */ From 11864f80cfd7a40277336bf921b7df979f878de8 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 21 Jan 2024 00:54:18 +1300 Subject: [PATCH 040/431] d->m: Make PK members appear real --- d2m/actions/edit-message.js | 18 ++- d2m/actions/register-pk-user.js | 139 ++++++++++++++++++++++++ d2m/actions/register-user.js | 5 +- d2m/actions/send-message.js | 12 +- d2m/actions/speedbump.js | 6 +- d2m/event-dispatcher.js | 36 +++--- db/migrations/0009-add-speedbump-id.sql | 1 + db/orm-defs.d.ts | 1 + types.d.ts | 20 ++++ 9 files changed, 215 insertions(+), 23 deletions(-) create mode 100644 d2m/actions/register-pk-user.js diff --git a/d2m/actions/edit-message.js b/d2m/actions/edit-message.js index 2a08526..d8c5f97 100644 --- a/d2m/actions/edit-message.js +++ b/d2m/actions/edit-message.js @@ -1,18 +1,32 @@ // @ts-check +const assert = require("assert").strict + const passthrough = require("../../passthrough") const {sync, db, select} = passthrough /** @type {import("../converters/edit-to-changes")} */ const editToChanges = sync.require("../converters/edit-to-changes") +/** @type {import("./register-pk-user")} */ +const registerPkUser = sync.require("./register-pk-user") /** @type {import("../../matrix/api")} */ const api = sync.require("../../matrix/api") /** * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message * @param {import("discord-api-types/v10").APIGuild} guild + * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel */ -async function editMessage(message, guild) { - const {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api) +async function editMessage(message, guild, row) { + let {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api) + + if (row && row.speedbump_webhook_id === message.webhook_id) { + // Handle the PluralKit public instance + if (row.speedbump_id === "466378653216014359") { + const root = await registerPkUser.fetchMessage(message.id) + assert(root.member) + senderMxid = await registerPkUser.ensureSimJoined(root.member, roomID) + } + } // 1. Replace all the things. for (const {oldID, newContent} of eventsToReplace) { diff --git a/d2m/actions/register-pk-user.js b/d2m/actions/register-pk-user.js new file mode 100644 index 0000000..f0fd492 --- /dev/null +++ b/d2m/actions/register-pk-user.js @@ -0,0 +1,139 @@ +// @ts-check + +const assert = require("assert") +const reg = require("../../matrix/read-registration") +const Ty = require("../../types") +const fetch = require("node-fetch").default + +const passthrough = require("../../passthrough") +const {discord, sync, db, select} = passthrough +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../../matrix/file")} */ +const file = sync.require("../../matrix/file") +/** @type {import("./register-user")} */ +const registerUser = sync.require("./register-user") + +/** + * A sim is an account that is being simulated by the bridge to copy events from the other side. + * @param {Ty.PkMember} member + * @returns mxid + */ +async function createSim(member) { + // Choose sim name + const simName = "_pk_" + member.id + const localpart = reg.ooye.namespace_prefix + simName + const mxid = `@${localpart}:${reg.ooye.server_name}` + + // Save chosen name in the database forever + db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(member.uuid, simName, localpart, mxid) + + // Register matrix user with that name + try { + await api.register(localpart) + } catch (e) { + // If user creation fails, manually undo the database change. Still isn't perfect, but should help. + // (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.) + db.prepare("DELETE FROM sim WHERE user_id = ?").run(member.uuid) + throw e + } + return mxid +} + +/** + * Ensure a sim is registered for the user. + * If there is already a sim, use that one. If there isn't one yet, register a new sim. + * @param {Ty.PkMember} member + * @returns {Promise} mxid + */ +async function ensureSim(member) { + let mxid = null + const existing = select("sim", "mxid", {user_id: member.uuid}).pluck().get() + if (existing) { + mxid = existing + } else { + mxid = await createSim(member) + } + return mxid +} + +/** + * Ensure a sim is registered for the user and is joined to the room. + * @param {Ty.PkMember} member + * @param {string} roomID + * @returns {Promise} mxid + */ +async function ensureSimJoined(member, roomID) { + // Ensure room ID is really an ID, not an alias + assert.ok(roomID[0] === "!") + + // Ensure user + const mxid = await ensureSim(member) + + // Ensure joined + const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get() + if (!existing) { + try { + await api.inviteToRoom(roomID, mxid) + await api.joinRoom(roomID, mxid) + } catch (e) { + if (e.message.includes("is already in the room.")) { + // Sweet! + } else { + throw e + } + } + db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid) + } + return mxid +} + +/** + * @param {Ty.PkMember} member + */ +async function memberToStateContent(member) { + const displayname = member.display_name || member.name + const avatar = member.avatar_url || member.webhook_avatar_url + + const content = { + displayname, + membership: "join", + "moe.cadence.ooye.pk_member": member + } + if (avatar) content.avatar_url = await file.uploadDiscordFileToMxc(avatar) + + return content +} + +/** + * Sync profile data for a sim user. This function follows the following process: + * 1. Join the sim to the room if needed + * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before + * 3. Compare against the previously known state content, which is helpfully stored in the database + * 4. If the state content has changed, send it to Matrix and update it in the database for next time + * @param {Ty.PkMember} member + * @returns {Promise} mxid of the updated sim + */ +async function syncUser(member, roomID) { + const mxid = await ensureSimJoined(member, roomID) + const content = await memberToStateContent(member) + const currentHash = registerUser._hashProfileContent(content) + const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() + // only do the actual sync if the hash has changed since we last looked + if (existingHash !== currentHash) { + await api.sendState(roomID, "m.room.member", mxid, content, mxid) + db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) + } + return mxid +} + +/** @returns {Promise<{member?: Ty.PkMember}>} */ +function fetchMessage(messageID) { + return fetch(`https://api.pluralkit.me/v2/messages/${messageID}`).then(res => res.json()) +} + +module.exports._memberToStateContent = memberToStateContent +module.exports.ensureSim = ensureSim +module.exports.ensureSimJoined = ensureSimJoined +module.exports.syncUser = syncUser +module.exports.fetchMessage = fetchMessage diff --git a/d2m/actions/register-user.js b/d2m/actions/register-user.js index b605c3a..8244fe2 100644 --- a/d2m/actions/register-user.js +++ b/d2m/actions/register-user.js @@ -123,7 +123,7 @@ async function memberToStateContent(user, member, guildID) { return content } -function hashProfileContent(content) { +function _hashProfileContent(content) { const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`) const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range return signedHash @@ -142,7 +142,7 @@ function hashProfileContent(content) { async function syncUser(user, member, guildID, roomID) { const mxid = await ensureSimJoined(user, roomID) const content = await memberToStateContent(user, member, guildID) - const currentHash = hashProfileContent(content) + const currentHash = _hashProfileContent(content) const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() // only do the actual sync if the hash has changed since we last looked if (existingHash !== currentHash) { @@ -179,6 +179,7 @@ async function syncAllUsersInRoom(roomID) { } module.exports._memberToStateContent = memberToStateContent +module.exports._hashProfileContent = _hashProfileContent module.exports.ensureSim = ensureSim module.exports.ensureSimJoined = ensureSimJoined module.exports.syncUser = syncUser diff --git a/d2m/actions/send-message.js b/d2m/actions/send-message.js index b59fc7f..8d02c43 100644 --- a/d2m/actions/send-message.js +++ b/d2m/actions/send-message.js @@ -10,6 +10,8 @@ const messageToEvent = sync.require("../converters/message-to-event") const api = sync.require("../../matrix/api") /** @type {import("./register-user")} */ const registerUser = sync.require("./register-user") +/** @type {import("./register-pk-user")} */ +const registerPkUser = sync.require("./register-pk-user") /** @type {import("../actions/create-room")} */ const createRoom = sync.require("../actions/create-room") /** @type {import("../../discord/utils")} */ @@ -18,8 +20,9 @@ const dUtils = sync.require("../../discord/utils") /** * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message * @param {import("discord-api-types/v10").APIGuild} guild + * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel */ -async function sendMessage(message, guild) { +async function sendMessage(message, guild, row) { const roomID = await createRoom.ensureRoom(message.channel_id) let senderMxid = null @@ -29,6 +32,13 @@ async function sendMessage(message, guild) { } else { // well, good enough... senderMxid = await registerUser.ensureSimJoined(message.author, roomID) } + } else if (row && row.speedbump_webhook_id === message.webhook_id) { + // Handle the PluralKit public instance + if (row.speedbump_id === "466378653216014359") { + const root = await registerPkUser.fetchMessage(message.id) + assert(root.member) // Member is null if member was deleted. We just got this message, so member surely exists. + senderMxid = await registerPkUser.syncUser(root.member, roomID) + } } const events = await messageToEvent.messageToEvent(message, guild, {}, {api}) diff --git a/d2m/actions/speedbump.js b/d2m/actions/speedbump.js index b56cbf6..ac1ce67 100644 --- a/d2m/actions/speedbump.js +++ b/d2m/actions/speedbump.js @@ -22,8 +22,10 @@ async function updateCache(channelID, speedbumpID, speedbumpChecked) { const now = Math.floor(Date.now() / 1000) if (speedbumpChecked && now - speedbumpChecked < SPEEDBUMP_UPDATE_FREQUENCY) return const webhooks = await discord.snow.webhook.getChannelWebhooks(channelID) - const found = webhooks.find(b => KNOWN_BOTS.has(b.application_id))?.application_id || null - db.prepare("UPDATE channel_room SET speedbump_id = ?, speedbump_checked = ? WHERE channel_id = ?").run(found, now, channelID) + const found = webhooks.find(b => KNOWN_BOTS.has(b.application_id)) + const foundApplication = found?.application_id + const foundWebhook = found?.id + db.prepare("UPDATE channel_room SET speedbump_id = ?, speedbump_webhook_id = ?, speedbump_checked = ? WHERE channel_id = ?").run(foundApplication, foundWebhook, now, channelID) } /** @type {Set} set of messageID */ diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 3a80d0a..6003152 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -236,22 +236,22 @@ module.exports = { */ async onMessageCreate(client, message) { if (message.author.username === "Deleted User") return // Nothing we can do for deleted users. - if (message.webhook_id) { - const row = select("webhook", "webhook_id", {webhook_id: message.webhook_id}).pluck().get() - if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. - } else { - const speedbumpID = select("channel_room", "speedbump_id", {channel_id: message.channel_id}).pluck().get() - if (speedbumpID) { - const affected = await speedbump.doSpeedbump(message.id) - if (affected) return - } - } const channel = client.channels.get(message.channel_id) if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages. const guild = client.guilds.get(channel.guild_id) assert(guild) - await sendMessage.sendMessage(message, guild), + const row = select("channel_room", ["speedbump_id", "speedbump_webhook_id"], {channel_id: message.channel_id}).get() + if (message.webhook_id) { + const row = select("webhook", "webhook_id", {webhook_id: message.webhook_id}).pluck().get() + if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. + } else if (row) { + const affected = await speedbump.doSpeedbump(message.id) + if (affected) return + } + + // @ts-ignore + await sendMessage.sendMessage(message, guild, row), await discordCommandHandler.execute(message, channel, guild) }, @@ -260,13 +260,16 @@ module.exports = { * @param {DiscordTypes.GatewayMessageUpdateDispatchData} data */ async onMessageUpdate(client, data) { + const row = select("channel_room", ["speedbump_id", "speedbump_webhook_id"], {channel_id: data.channel_id}).get() if (data.webhook_id) { const row = select("webhook", "webhook_id", {webhook_id: data.webhook_id}).pluck().get() - if (row) { - // The update was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. - return - } + if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. + } else if (row) { + // Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from. + const affected = await speedbump.doSpeedbump(data.id) + if (affected) return } + // Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes. // If the message content is a string then it includes all interesting fields and is meaningful. if (typeof data.content === "string") { @@ -277,7 +280,8 @@ module.exports = { if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages. const guild = client.guilds.get(channel.guild_id) assert(guild) - await editMessage.editMessage(message, guild) + // @ts-ignore + await editMessage.editMessage(message, guild, row) } }, diff --git a/db/migrations/0009-add-speedbump-id.sql b/db/migrations/0009-add-speedbump-id.sql index e971146..67a415c 100644 --- a/db/migrations/0009-add-speedbump-id.sql +++ b/db/migrations/0009-add-speedbump-id.sql @@ -1,6 +1,7 @@ BEGIN TRANSACTION; ALTER TABLE channel_room ADD COLUMN speedbump_id TEXT; +ALTER TABLE channel_room ADD COLUMN speedbump_webhook_id TEXT; ALTER TABLE channel_room ADD COLUMN speedbump_checked INTEGER; COMMIT; diff --git a/db/orm-defs.d.ts b/db/orm-defs.d.ts index f0f9a67..540c7a6 100644 --- a/db/orm-defs.d.ts +++ b/db/orm-defs.d.ts @@ -8,6 +8,7 @@ export type Models = { custom_avatar: string | null last_bridged_pin_timestamp: number | null speedbump_id: string | null + speedbump_webhook_id: string | null speedbump_checked: number | null } diff --git a/types.d.ts b/types.d.ts index 9e9d72b..daf62ad 100644 --- a/types.d.ts +++ b/types.d.ts @@ -34,6 +34,26 @@ export type WebhookCreds = { token: string } +export type PkMember = { + id: string + uuid: string + name: string + display_name: string | null + color: string | null + birthday: string | null + pronouns: string | null + avatar_url: string | null + webhook_avatar_url: string | null + banner: string | null + description: string | null + created: string | null + keep_proxy: boolean + tts: boolean + autoproxy_enabled: boolean | null + message_count: number | null + last_message_timestamp: string +} + export namespace Event { export type Outer = { type: string From 6a06dc14ceccb094b5d09282c511ba822e3a391f Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 22 Jan 2024 22:05:59 +1300 Subject: [PATCH 041/431] Fix all rooms being affected by speedbump --- d2m/actions/delete-message.js | 4 ++-- d2m/actions/speedbump.js | 7 +++---- d2m/event-dispatcher.js | 12 ++++++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/d2m/actions/delete-message.js b/d2m/actions/delete-message.js index 496d827..4386ae5 100644 --- a/d2m/actions/delete-message.js +++ b/d2m/actions/delete-message.js @@ -11,7 +11,7 @@ const speedbump = sync.require("./speedbump") * @param {import("discord-api-types/v10").GatewayMessageDeleteDispatchData} data */ async function deleteMessage(data) { - const row = select("channel_room", ["room_id", "speedbump_id", "speedbump_checked"], {channel_id: data.channel_id}).get() + const row = select("channel_room", ["room_id", "speedbump_checked"], {channel_id: data.channel_id}).get() if (!row) return const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all() @@ -22,7 +22,7 @@ async function deleteMessage(data) { await api.redactEvent(row.room_id, eventID) } - speedbump.updateCache(data.channel_id, row.speedbump_id, row.speedbump_checked) + speedbump.updateCache(data.channel_id, row.speedbump_checked) } /** diff --git a/d2m/actions/speedbump.js b/d2m/actions/speedbump.js index ac1ce67..f49a378 100644 --- a/d2m/actions/speedbump.js +++ b/d2m/actions/speedbump.js @@ -15,12 +15,11 @@ const KNOWN_BOTS = new Set([ /** * Fetch new speedbump data for the channel and put it in the database as cache * @param {string} channelID - * @param {string?} speedbumpID - * @param {number?} speedbumpChecked + * @param {number?} lastChecked */ -async function updateCache(channelID, speedbumpID, speedbumpChecked) { +async function updateCache(channelID, lastChecked) { const now = Math.floor(Date.now() / 1000) - if (speedbumpChecked && now - speedbumpChecked < SPEEDBUMP_UPDATE_FREQUENCY) return + if (lastChecked && now - lastChecked < SPEEDBUMP_UPDATE_FREQUENCY) return const webhooks = await discord.snow.webhook.getChannelWebhooks(channelID) const found = webhooks.find(b => KNOWN_BOTS.has(b.application_id)) const foundApplication = found?.application_id diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 6003152..c630bfb 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -241,11 +241,13 @@ module.exports = { const guild = client.guilds.get(channel.guild_id) assert(guild) - const row = select("channel_room", ["speedbump_id", "speedbump_webhook_id"], {channel_id: message.channel_id}).get() if (message.webhook_id) { const row = select("webhook", "webhook_id", {webhook_id: message.webhook_id}).pluck().get() if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. - } else if (row) { + } + + const row = select("channel_room", ["speedbump_id", "speedbump_webhook_id"], {channel_id: message.channel_id}).get() + if (row && row.speedbump_id) { const affected = await speedbump.doSpeedbump(message.id) if (affected) return } @@ -260,11 +262,13 @@ module.exports = { * @param {DiscordTypes.GatewayMessageUpdateDispatchData} data */ async onMessageUpdate(client, data) { - const row = select("channel_room", ["speedbump_id", "speedbump_webhook_id"], {channel_id: data.channel_id}).get() if (data.webhook_id) { const row = select("webhook", "webhook_id", {webhook_id: data.webhook_id}).pluck().get() if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. - } else if (row) { + } + + const row = select("channel_room", ["speedbump_id", "speedbump_webhook_id"], {channel_id: data.channel_id}).get() + if (row) { // Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from. const affected = await speedbump.doSpeedbump(data.id) if (affected) return From a71c9515ec17d6cbc873b2249f3e12c9a0c4138b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 22 Jan 2024 22:30:31 +1300 Subject: [PATCH 042/431] Include system data on PK profiles --- d2m/actions/edit-message.js | 2 +- d2m/actions/register-pk-user.js | 45 +++++++++++++++++---------------- d2m/actions/send-message.js | 9 +++++-- types.d.ts | 18 +++++++++++++ 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/d2m/actions/edit-message.js b/d2m/actions/edit-message.js index d8c5f97..d52fcbd 100644 --- a/d2m/actions/edit-message.js +++ b/d2m/actions/edit-message.js @@ -24,7 +24,7 @@ async function editMessage(message, guild, row) { if (row.speedbump_id === "466378653216014359") { const root = await registerPkUser.fetchMessage(message.id) assert(root.member) - senderMxid = await registerPkUser.ensureSimJoined(root.member, roomID) + senderMxid = await registerPkUser.ensureSimJoined(root, roomID) } } diff --git a/d2m/actions/register-pk-user.js b/d2m/actions/register-pk-user.js index f0fd492..da94f01 100644 --- a/d2m/actions/register-pk-user.js +++ b/d2m/actions/register-pk-user.js @@ -16,17 +16,17 @@ const registerUser = sync.require("./register-user") /** * A sim is an account that is being simulated by the bridge to copy events from the other side. - * @param {Ty.PkMember} member + * @param {Ty.PkMessage} pkMessage * @returns mxid */ -async function createSim(member) { +async function createSim(pkMessage) { // Choose sim name - const simName = "_pk_" + member.id + const simName = "_pk_" + pkMessage.member.id const localpart = reg.ooye.namespace_prefix + simName const mxid = `@${localpart}:${reg.ooye.server_name}` // Save chosen name in the database forever - db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(member.uuid, simName, localpart, mxid) + db.prepare("INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES (?, ?, ?, ?)").run(pkMessage.member.uuid, simName, localpart, mxid) // Register matrix user with that name try { @@ -34,7 +34,7 @@ async function createSim(member) { } catch (e) { // If user creation fails, manually undo the database change. Still isn't perfect, but should help. // (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.) - db.prepare("DELETE FROM sim WHERE user_id = ?").run(member.uuid) + db.prepare("DELETE FROM sim WHERE user_id = ?").run(pkMessage.member.uuid) throw e } return mxid @@ -43,32 +43,32 @@ async function createSim(member) { /** * Ensure a sim is registered for the user. * If there is already a sim, use that one. If there isn't one yet, register a new sim. - * @param {Ty.PkMember} member + * @param {Ty.PkMessage} pkMessage * @returns {Promise} mxid */ -async function ensureSim(member) { +async function ensureSim(pkMessage) { let mxid = null - const existing = select("sim", "mxid", {user_id: member.uuid}).pluck().get() + const existing = select("sim", "mxid", {user_id: pkMessage.member.uuid}).pluck().get() if (existing) { mxid = existing } else { - mxid = await createSim(member) + mxid = await createSim(pkMessage) } return mxid } /** * Ensure a sim is registered for the user and is joined to the room. - * @param {Ty.PkMember} member + * @param {Ty.PkMessage} pkMessage * @param {string} roomID * @returns {Promise} mxid */ -async function ensureSimJoined(member, roomID) { +async function ensureSimJoined(pkMessage, roomID) { // Ensure room ID is really an ID, not an alias assert.ok(roomID[0] === "!") // Ensure user - const mxid = await ensureSim(member) + const mxid = await ensureSim(pkMessage) // Ensure joined const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get() @@ -89,16 +89,17 @@ async function ensureSimJoined(member, roomID) { } /** - * @param {Ty.PkMember} member + * @param {Ty.PkMessage} pkMessage */ -async function memberToStateContent(member) { - const displayname = member.display_name || member.name - const avatar = member.avatar_url || member.webhook_avatar_url +async function memberToStateContent(pkMessage) { + const systemname = pkMessage.system.tag || "" + const displayname = (pkMessage.member.display_name || pkMessage.member.name) + systemname + const avatar = pkMessage.member.avatar_url || pkMessage.member.webhook_avatar_url || pkMessage.system.avatar_url const content = { displayname, membership: "join", - "moe.cadence.ooye.pk_member": member + "moe.cadence.ooye.pk_member": pkMessage.member } if (avatar) content.avatar_url = await file.uploadDiscordFileToMxc(avatar) @@ -111,12 +112,12 @@ async function memberToStateContent(member) { * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before * 3. Compare against the previously known state content, which is helpfully stored in the database * 4. If the state content has changed, send it to Matrix and update it in the database for next time - * @param {Ty.PkMember} member + * @param {Ty.PkMessage} pkMessage * @returns {Promise} mxid of the updated sim */ -async function syncUser(member, roomID) { - const mxid = await ensureSimJoined(member, roomID) - const content = await memberToStateContent(member) +async function syncUser(pkMessage, roomID) { + const mxid = await ensureSimJoined(pkMessage, roomID) + const content = await memberToStateContent(pkMessage) const currentHash = registerUser._hashProfileContent(content) const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() // only do the actual sync if the hash has changed since we last looked @@ -127,7 +128,7 @@ async function syncUser(member, roomID) { return mxid } -/** @returns {Promise<{member?: Ty.PkMember}>} */ +/** @returns {Promise} */ function fetchMessage(messageID) { return fetch(`https://api.pluralkit.me/v2/messages/${messageID}`).then(res => res.json()) } diff --git a/d2m/actions/send-message.js b/d2m/actions/send-message.js index 8d02c43..72011a1 100644 --- a/d2m/actions/send-message.js +++ b/d2m/actions/send-message.js @@ -36,8 +36,13 @@ async function sendMessage(message, guild, row) { // Handle the PluralKit public instance if (row.speedbump_id === "466378653216014359") { const root = await registerPkUser.fetchMessage(message.id) - assert(root.member) // Member is null if member was deleted. We just got this message, so member surely exists. - senderMxid = await registerPkUser.syncUser(root.member, roomID) + // Member is null if member was deleted. We just got this message, so member surely exists. + if (!root.member) { + const e = new Error("PK API did not return a member") + e["response"] = root + throw e + } + senderMxid = await registerPkUser.syncUser(root, roomID) } } diff --git a/types.d.ts b/types.d.ts index daf62ad..a01241c 100644 --- a/types.d.ts +++ b/types.d.ts @@ -34,6 +34,19 @@ export type WebhookCreds = { token: string } +export type PkSystem = { + id: string + uuid: string + name: string | null + description: string | null + tag: string | null + pronouns: string | null + avatar_url: string | null + banner: string | null + color: string | null + created: string | null +} + export type PkMember = { id: string uuid: string @@ -54,6 +67,11 @@ export type PkMember = { last_message_timestamp: string } +export type PkMessage = { + system: PkSystem + member: PkMember +} + export namespace Event { export type Outer = { type: string From 4591b5ae034b573b82f4408b44598d8f058bdf96 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 22 Jan 2024 23:10:33 +1300 Subject: [PATCH 043/431] m->d: Fix glitched mention when Element disambigs --- d2m/actions/register-pk-user.js | 6 +++-- m2d/converters/event-to-message.js | 4 +++- m2d/converters/event-to-message.test.js | 31 +++++++++++++++++++++++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/d2m/actions/register-pk-user.js b/d2m/actions/register-pk-user.js index da94f01..5bce6ab 100644 --- a/d2m/actions/register-pk-user.js +++ b/d2m/actions/register-pk-user.js @@ -92,8 +92,10 @@ async function ensureSimJoined(pkMessage, roomID) { * @param {Ty.PkMessage} pkMessage */ async function memberToStateContent(pkMessage) { - const systemname = pkMessage.system.tag || "" - const displayname = (pkMessage.member.display_name || pkMessage.member.name) + systemname + let displayname = (pkMessage.member.display_name || pkMessage.member.name) + if (pkMessage.system.tag) { + displayname = displayname + " " + pkMessage.system.tag + } const avatar = pkMessage.member.avatar_url || pkMessage.member.webhook_avatar_url || pkMessage.system.avatar_url const content = { diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 878dcf5..3b1a7d2 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -109,6 +109,8 @@ turndownService.addRule("inlineLink", { const href = node.getAttribute("href") let brackets = ["", ""] if (href.startsWith("https://matrix.to")) brackets = ["<", ">"] + if (href.startsWith("https://matrix.to/#/@")) content = "@" + content + content = content.replace(/ @.*/, "") return "[" + content + "](" + brackets[0] + href + brackets[1] + ")" } }) @@ -621,7 +623,7 @@ async function eventToMessage(event, guild, di) { content = displayNameRunoff + replyLine + content // Handling written @mentions: we need to look for candidate Discord members to join to the room - let writtenMentionMatch = content.match(/(?:^|[^"<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // /d flag for indices requires node.js 16+ + let writtenMentionMatch = content.match(/(?:^|[^"[<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // /d flag for indices requires node.js 16+ if (writtenMentionMatch) { const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]}) if (results[0]) { diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 4f1c1dd..ddb5c58 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -1925,7 +1925,34 @@ test("event2message: mentioning matrix users works", async t => { messagesToEdit: [], messagesToSend: [{ username: "cadence [they]", - content: "I'm just [▲]() testing mentions", + content: "I'm just [@▲]() testing mentions", + avatar_url: undefined + }] + } + ) +}) + +test("event2message: mentioning matrix users works even when Element disambiguates the user", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "unascribed @unascribed:sleeping.town: if you want to run some experimental software, `11864f80cf` branch of OOYE has _vastly_ improved handling of PluralKit users. feel free to try it out, if you find bugs I'd appreciate you letting me know (just tag me at the place in chat where something went wrong)", + format: "org.matrix.custom.html", + formatted_body: "unascribed @unascribed:sleeping.town: if you want to run some experimental software, 11864f80cf branch of OOYE has vastly improved handling of PluralKit users. feel free to try it out, if you find bugs I'd appreciate you letting me know (just tag me at the place in chat where something went wrong)" + }, + event_id: "$17qTyvkDykSp_4Wkjeuh9Y6j9hPe20ZY_E6V3UKAyUE", + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "[@unascribed]() if you want to run some experimental software, `11864f80cf` branch of OOYE has _vastly_ improved handling of PluralKit users. feel free to try it out, if you find bugs I'd appreciate you letting me know (just tag me at the place in chat where something went wrong)", avatar_url: undefined }] } @@ -2264,7 +2291,7 @@ test("event2message: colon after mentions is stripped", async t => { messagesToEdit: [], messagesToSend: [{ username: "cadence [they]", - content: "<@114147806469554185> hey, I'm just [▲]() testing mentions", + content: "<@114147806469554185> hey, I'm just [@▲]() testing mentions", avatar_url: undefined }] } From c084aa0156b548d69a57cfe93dd2455952243ffe Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 22 Jan 2024 23:36:19 +1300 Subject: [PATCH 044/431] Add the @ sign in the other order --- m2d/converters/event-to-message.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 3b1a7d2..6d0232c 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -108,9 +108,9 @@ turndownService.addRule("inlineLink", { if (node.getAttribute("data-channel-id")) return `<#${node.getAttribute("data-channel-id")}>` const href = node.getAttribute("href") let brackets = ["", ""] - if (href.startsWith("https://matrix.to")) brackets = ["<", ">"] - if (href.startsWith("https://matrix.to/#/@")) content = "@" + content content = content.replace(/ @.*/, "") + if (href.startsWith("https://matrix.to")) brackets = ["<", ">"] + if (href.startsWith("https://matrix.to/#/@") && content[0] !== "@") content = "@" + content return "[" + content + "](" + brackets[0] + href + brackets[1] + ")" } }) From 3d87bd9da58aec774bf0224082ae67ce97323541 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 30 Jan 2024 22:01:06 +1300 Subject: [PATCH 045/431] PK: Use webhook name as bridged name --- d2m/actions/register-pk-user.js | 25 ++++++++++++++++--------- d2m/actions/send-message.js | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/d2m/actions/register-pk-user.js b/d2m/actions/register-pk-user.js index 5bce6ab..002bfb9 100644 --- a/d2m/actions/register-pk-user.js +++ b/d2m/actions/register-pk-user.js @@ -14,6 +14,13 @@ const file = sync.require("../../matrix/file") /** @type {import("./register-user")} */ const registerUser = sync.require("./register-user") +/** + * @typedef WebhookAuthor Discord API message->author. A webhook as an author. + * @prop {string} username + * @prop {string?} avatar + * @prop {string} id + */ + /** * A sim is an account that is being simulated by the bridge to copy events from the other side. * @param {Ty.PkMessage} pkMessage @@ -90,16 +97,15 @@ async function ensureSimJoined(pkMessage, roomID) { /** * @param {Ty.PkMessage} pkMessage + * @param {WebhookAuthor} author */ -async function memberToStateContent(pkMessage) { - let displayname = (pkMessage.member.display_name || pkMessage.member.name) - if (pkMessage.system.tag) { - displayname = displayname + " " + pkMessage.system.tag - } - const avatar = pkMessage.member.avatar_url || pkMessage.member.webhook_avatar_url || pkMessage.system.avatar_url +async function memberToStateContent(pkMessage, author) { + // We prefer to use the member's avatar URL data since the image upload can be cached across channels, + // unlike the userAvatar URL which is unique per channel, due to the webhook ID being in the URL. + const avatar = pkMessage.member.avatar_url || pkMessage.member.webhook_avatar_url || pkMessage.system.avatar_url || file.userAvatar(author) const content = { - displayname, + displayname: author.username, membership: "join", "moe.cadence.ooye.pk_member": pkMessage.member } @@ -114,12 +120,13 @@ async function memberToStateContent(pkMessage) { * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before * 3. Compare against the previously known state content, which is helpfully stored in the database * 4. If the state content has changed, send it to Matrix and update it in the database for next time + * @param {WebhookAuthor} author * @param {Ty.PkMessage} pkMessage * @returns {Promise} mxid of the updated sim */ -async function syncUser(pkMessage, roomID) { +async function syncUser(author, pkMessage, roomID) { const mxid = await ensureSimJoined(pkMessage, roomID) - const content = await memberToStateContent(pkMessage) + const content = await memberToStateContent(pkMessage, author) const currentHash = registerUser._hashProfileContent(content) const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() // only do the actual sync if the hash has changed since we last looked diff --git a/d2m/actions/send-message.js b/d2m/actions/send-message.js index 72011a1..1d25430 100644 --- a/d2m/actions/send-message.js +++ b/d2m/actions/send-message.js @@ -42,7 +42,7 @@ async function sendMessage(message, guild, row) { e["response"] = root throw e } - senderMxid = await registerPkUser.syncUser(root, roomID) + senderMxid = await registerPkUser.syncUser(message.author, root, roomID) } } From f48c1f3f31ee123212bfa64c36a5a0a6ce76b7cb Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 31 Jan 2024 13:09:39 +1300 Subject: [PATCH 046/431] PK: Fix mentions/replies using UUID --- d2m/actions/register-pk-user.js | 4 ++++ db/migrations/0010-add-sim-proxy.sql | 5 ++++ db/orm-defs.d.ts | 6 +++++ db/orm.js | 9 +++++-- db/orm.test.js | 9 +++++++ m2d/converters/event-to-message.js | 16 ++++++++++--- m2d/converters/event-to-message.test.js | 32 ++++++++++++++++++++++++- test/ooye-test-data.sql | 6 ++++- types.d.ts | 1 + 9 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 db/migrations/0010-add-sim-proxy.sql diff --git a/d2m/actions/register-pk-user.js b/d2m/actions/register-pk-user.js index 002bfb9..003453d 100644 --- a/d2m/actions/register-pk-user.js +++ b/d2m/actions/register-pk-user.js @@ -122,10 +122,14 @@ async function memberToStateContent(pkMessage, author) { * 4. If the state content has changed, send it to Matrix and update it in the database for next time * @param {WebhookAuthor} author * @param {Ty.PkMessage} pkMessage + * @param {string} roomID * @returns {Promise} mxid of the updated sim */ async function syncUser(author, pkMessage, roomID) { const mxid = await ensureSimJoined(pkMessage, roomID) + // Update the sim_proxy table, so mentions can look up the original sender later + db.prepare("INSERT OR IGNORE INTO sim_proxy (user_id, proxy_owner_id) VALUES (?, ?)").run(pkMessage.member.id, pkMessage.sender) + // Sync the member state const content = await memberToStateContent(pkMessage, author) const currentHash = registerUser._hashProfileContent(content) const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get() diff --git a/db/migrations/0010-add-sim-proxy.sql b/db/migrations/0010-add-sim-proxy.sql new file mode 100644 index 0000000..d7dd0a1 --- /dev/null +++ b/db/migrations/0010-add-sim-proxy.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS sim_proxy ( + user_id TEXT NOT NULL, + proxy_owner_id TEXT NOT NULL, + PRIMARY KEY(user_id) +) WITHOUT ROWID; diff --git a/db/orm-defs.d.ts b/db/orm-defs.d.ts index 540c7a6..35fea51 100644 --- a/db/orm-defs.d.ts +++ b/db/orm-defs.d.ts @@ -63,6 +63,11 @@ export type Models = { hashed_profile_content: number } + sim_proxy: { + user_id: string + proxy_owner_id: string + } + webhook: { channel_id: string webhook_id: string @@ -100,3 +105,4 @@ export type Prepared = { export type AllKeys = U extends any ? keyof U : never export type PickTypeOf> = T extends { [k in K]?: any } ? T[K] : never export type Merge = {[x in AllKeys]: PickTypeOf} +export type Nullable = {[k in keyof T]: T[k] | null} diff --git a/db/orm.js b/db/orm.js index d9cc1e8..c6cab96 100644 --- a/db/orm.js +++ b/db/orm.js @@ -38,6 +38,8 @@ class From { /** @private @type {Table[]} */ this.tables = [table] /** @private */ + this.directions = [] + /** @private */ this.sql = "" /** @private */ this.cols = [] @@ -53,12 +55,14 @@ class From { * @template {keyof U.Models} Table2 * @param {Table2} table * @param {Col & (keyof U.Models[Table2])} col + * @param {"inner" | "left"} [direction] */ - join(table, col) { + join(table, col, direction = "inner") { /** @type {From>} */ // @ts-ignore const r = this r.tables.push(table) + r.directions.push(direction.toUpperCase()) r.using.push(col) return r } @@ -112,7 +116,8 @@ class From { for (let i = 1; i < this.tables.length; i++) { const table = this.tables[i] const col = this.using[i-1] - sql += `INNER JOIN ${table} USING (${col}) ` + const direction = this.directions[i-1] + sql += `${direction} JOIN ${table} USING (${col}) ` } sql += this.sql /** @type {U.Prepared, Col>>} */ diff --git a/db/orm.test.js b/db/orm.test.js index 36e95c2..066eabb 100644 --- a/db/orm.test.js +++ b/db/orm.test.js @@ -44,3 +44,12 @@ test("orm: from: where and pluck works", t => { const subtypes = from("event_message").where({message_id: "1141501302736695316"}).pluck("event_subtype").all() t.deepEqual(subtypes.sort(), ["m.image", "m.text"]) }) + +test("orm: from: join direction works", t => { + const hasOwner = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({sim_name: "_pk_zoego"}).get() + t.deepEqual(hasOwner, {user_id: "43d378d5-1183-47dc-ab3c-d14e21c3fe58", proxy_owner_id: "196188877885538304"}) + const hasNoOwner = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({sim_name: "crunch_god"}).get() + t.deepEqual(hasNoOwner, {user_id: "820865262526005258", proxy_owner_id: null}) + const hasNoOwnerInner = from("sim").join("sim_proxy", "user_id", "inner").select("user_id", "proxy_owner_id").where({sim_name: "crunch_god"}).get() + t.deepEqual(hasNoOwnerInner, null) +}) diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 6d0232c..a48fa75 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -238,6 +238,16 @@ function splitDisplayName(displayName) { } } +/** + * Convert a Matrix user ID into a Discord user ID for mentioning, where if the user is a PK proxy, it will mention the proxy owner. + * @param {string} mxid + */ +function getUserOrProxyOwnerID(mxid) { + const row = from("sim").join("sim_proxy", "user_id", "left").select("user_id", "proxy_owner_id").where({mxid}).get() + if (!row) return null + return row.proxy_owner_id || row.user_id +} + /** * At the time of this executing, we know what the end of message emojis are, and we know that at least one of them is unknown. * This function will strip them from the content and generate the correct pending file of the sprite sheet. @@ -444,11 +454,11 @@ async function eventToMessage(event, guild, di) { replyLine += `https://discord.com/channels/${guild.id}/${row.channel_id}/${row.message_id} ` } const sender = repliedToEvent.sender - const authorID = select("sim", "user_id", {mxid: repliedToEvent.sender}).pluck().get() + const authorID = getUserOrProxyOwnerID(sender) if (authorID) { replyLine += `<@${authorID}>` } else { - let senderName = select("member_cache", "displayname", {mxid: repliedToEvent.sender}).pluck().get() + let senderName = select("member_cache", "displayname", {mxid: sender}).pluck().get() if (!senderName) { const match = sender.match(/@([^:]*)/) assert(match) @@ -497,7 +507,7 @@ async function eventToMessage(event, guild, di) { mxid = decodeURIComponent(mxid) if (mxUtils.eventSenderIsFromDiscord(mxid)) { // Handle mention of an OOYE sim user by their mxid - const userID = select("sim", "user_id", {mxid: mxid}).pluck().get() + const userID = getUserOrProxyOwnerID(mxid) if (!userID) return whole return `${attributeValue} data-user-id="${userID}">` } else { diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index ddb5c58..378e9f0 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -1869,7 +1869,6 @@ test("event2message: mentioning discord users works", async t => { ) }) - test("event2message: mentioning discord users works when URL encoded", async t => { t.deepEqual( await eventToMessage({ @@ -1901,6 +1900,37 @@ test("event2message: mentioning discord users works when URL encoded", async t = ) }) +test("event2message: mentioning PK discord users works", async t => { + t.deepEqual( + await eventToMessage({ + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: `I'm just Azalea testing mentions` + }, + event_id: "$g07oYSZFWBkxohNEfywldwgcWj1hbhDzQ1sBAKvqOOU", + origin_server_ts: 1688301929913, + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe", + sender: "@cadence:cadence.moe", + type: "m.room.message", + unsigned: { + age: 405299 + } + }), + { + ensureJoined: [], + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "I'm just <@196188877885538304> testing mentions", + avatar_url: undefined + }] + } + ) +}) + test("event2message: mentioning matrix users works", async t => { t.deepEqual( await eventToMessage({ diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 8da0128..f1b720f 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -21,7 +21,11 @@ INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES ('112890272819507200', '.wing.', '_ooye_.wing.', '@_ooye_.wing.:cadence.moe'), ('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'), ('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe'), -('1109360903096369153', 'amanda', '_ooye_amanda', '@_ooye_amanda:cadence.moe'); +('1109360903096369153', 'amanda', '_ooye_amanda', '@_ooye_amanda:cadence.moe'), +('43d378d5-1183-47dc-ab3c-d14e21c3fe58', '_pk_zoego', '_ooye__pk_zoego', '@_ooye__pk_zoego:cadence.moe'); + +INSERT INTO sim_proxy (user_id, proxy_owner_id) VALUES +('43d378d5-1183-47dc-ab3c-d14e21c3fe58', '196188877885538304'); INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES ('@_ooye_bojack_horseman:cadence.moe', '!hYnGGlPHlbujVVfktC:cadence.moe', NULL); diff --git a/types.d.ts b/types.d.ts index a01241c..e9946d7 100644 --- a/types.d.ts +++ b/types.d.ts @@ -70,6 +70,7 @@ export type PkMember = { export type PkMessage = { system: PkSystem member: PkMember + sender: string } export namespace Event { From 6c3164edd687d22577df8a259ce45da32ec144b9 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 1 Feb 2024 16:38:17 +1300 Subject: [PATCH 047/431] m->d: Emoji sprite sheet supports APNG --- m2d/converters/emoji-sheet.js | 124 ++++++++++++++++++----------- m2d/converters/emoji-sheet.test.js | 55 +++++++++++++ package-lock.json | 8 +- package.json | 2 +- test/test.js | 1 + 5 files changed, 138 insertions(+), 52 deletions(-) create mode 100644 m2d/converters/emoji-sheet.test.js diff --git a/m2d/converters/emoji-sheet.js b/m2d/converters/emoji-sheet.js index c85176b..9e8703d 100644 --- a/m2d/converters/emoji-sheet.js +++ b/m2d/converters/emoji-sheet.js @@ -4,6 +4,7 @@ const assert = require("assert").strict const {pipeline} = require("stream").promises const sharp = require("sharp") const {GIFrame} = require("giframe") +const {PNG} = require("pngjs") const utils = require("./utils") const fetch = require("node-fetch").default const streamMimeType = require("stream-mime-type") @@ -18,57 +19,23 @@ const IMAGES_ACROSS = Math.floor(RESULT_WIDTH / SIZE) * @returns {Promise} PNG image */ async function compositeMatrixEmojis(mxcs) { - let buffers = await Promise.all(mxcs.map(async mxc => { + const buffers = await Promise.all(mxcs.map(async mxc => { const abortController = new AbortController() - try { - const url = utils.getPublicUrlForMxc(mxc) - assert(url) + const url = utils.getPublicUrlForMxc(mxc) + assert(url) - /** @type {import("node-fetch").Response} res */ - // If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing. - // If we were using connection pooling, we would be forced to download the entire GIF. - // So we set no agent to ensure we are not connection pooling. - // @ts-ignore the signal is slightly different from the type it wants (still works fine) - const res = await fetch(url, {agent: false, signal: abortController.signal}) - const {stream, mime} = await streamMimeType.getMimeType(res.body) - assert(["image/png", "image/jpeg", "image/webp", "image/gif"].includes(mime), `Mime type ${mime} is impossible for emojis`) - - if (mime === "image/png" || mime === "image/jpeg" || mime === "image/webp") { - /** @type {{info: sharp.OutputInfo, buffer: Buffer}} */ - const result = await new Promise((resolve, reject) => { - const transformer = sharp() - .resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}}) - .png({compressionLevel: 0}) - .toBuffer((err, buffer, info) => { - /* c8 ignore next */ - if (err) return reject(err) - resolve({info, buffer}) - }) - pipeline( - stream, - transformer - ) - }) - return result.buffer - - } else if (mime === "image/gif") { - const giframe = new GIFrame(0) - stream.on("data", chunk => { - giframe.feed(chunk) - }) - const frame = await giframe.getFrame() - - const buffer = await sharp(frame.pixels, {raw: {width: frame.width, height: frame.height, channels: 4}}) - .resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}}) - .png({compressionLevel: 0}) - .toBuffer({resolveWithObject: true}) - return buffer.data - - } - } finally { + /** @type {import("node-fetch").Response} */ + // If it turns out to be a GIF, we want to abandon the connection without downloading the whole thing. + // If we were using connection pooling, we would be forced to download the entire GIF. + // So we set no agent to ensure we are not connection pooling. + // @ts-ignore the signal is slightly different from the type it wants (still works fine) + const res = await fetch(url, {agent: false, signal: abortController.signal}) + return convertImageStream(res.body, () => { abortController.abort() - } + res.body.pause() + res.body.emit("end") + }) })) // Calculate the size of the final composited image @@ -98,4 +65,67 @@ async function compositeMatrixEmojis(mxcs) { return output.data } +/** + * @param {import("node-fetch").Response["body"]} streamIn + * @param {() => any} stopStream + * @returns {Promise} Uncompressed PNG image + */ +async function convertImageStream(streamIn, stopStream) { + const {stream, mime} = await streamMimeType.getMimeType(streamIn) + assert(["image/png", "image/jpeg", "image/webp", "image/gif", "image/apng"].includes(mime), `Mime type ${mime} is impossible for emojis`) + + try { + if (mime === "image/png" || mime === "image/jpeg" || mime === "image/webp") { + /** @type {{info: sharp.OutputInfo, buffer: Buffer}} */ + const result = await new Promise((resolve, reject) => { + const transformer = sharp() + .resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}}) + .png({compressionLevel: 0}) + .toBuffer((err, buffer, info) => { + /* c8 ignore next */ + if (err) return reject(err) + resolve({info, buffer}) + }) + pipeline( + stream, + transformer + ) + }) + return result.buffer + + } else if (mime === "image/gif") { + const giframe = new GIFrame(0) + stream.on("data", chunk => { + giframe.feed(chunk) + }) + const frame = await giframe.getFrame() + stopStream() + + const buffer = await sharp(frame.pixels, {raw: {width: frame.width, height: frame.height, channels: 4}}) + .resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}}) + .png({compressionLevel: 0}) + .toBuffer({resolveWithObject: true}) + return buffer.data + + } else if (mime === "image/apng") { + const png = new PNG({maxFrames: 1}) + // @ts-ignore + stream.pipe(png) + /** @type {Buffer} */ // @ts-ignore + const frame = await new Promise(resolve => png.on("parsed", resolve)) + stopStream() + + const buffer = await sharp(frame, {raw: {width: png.width, height: png.height, channels: 4}}) + .resize(SIZE, SIZE, {fit: "contain", background: {r: 0, g: 0, b: 0, alpha: 0}}) + .png({compressionLevel: 0}) + .toBuffer({resolveWithObject: true}) + return buffer.data + + } + } finally { + stopStream() + } +} + module.exports.compositeMatrixEmojis = compositeMatrixEmojis +module.exports._convertImageStream = convertImageStream diff --git a/m2d/converters/emoji-sheet.test.js b/m2d/converters/emoji-sheet.test.js new file mode 100644 index 0000000..f75fafc --- /dev/null +++ b/m2d/converters/emoji-sheet.test.js @@ -0,0 +1,55 @@ +const assert = require("assert").strict +const {test} = require("supertape") +const {_convertImageStream} = require("./emoji-sheet") +const fetch = require("node-fetch") +const {Transform} = require("stream").Transform + +/* c8 ignore next 7 */ +function slow() { + if (process.argv.includes("--slow")) { + return test + } else { + return test.skip + } +} + +class Meter extends Transform { + bytes = 0 + + _transform(chunk, encoding, cb) { + this.bytes += chunk.length + this.push(chunk) + cb() + } +} + +/** + * @param {import("supertape").Test} t + * @param {string} url + * @param {number} totalSize + */ +async function runSingleTest(t, url, totalSize) { + const abortController = new AbortController() + const res = await fetch("https://ezgif.com/images/format-demo/butterfly.png", {agent: false, signal: abortController.signal}) + const meter = new Meter() + const p = res.body.pipe(meter) + const result = await _convertImageStream(p, () => { + abortController.abort() + res.body.pause() + res.body.emit("end") + }) + t.equal(result.subarray(1, 4).toString("ascii"), "PNG", `result was not a PNG file: ${result.toString("base64")}`) + if (meter.bytes < totalSize / 4) { // should download less than 25% of each file + t.pass("intentionally read partial file") + } else { + t.fail(`read more than 25% of file, read: ${meter.bytes}, total: ${totalSize}`) + } +} + +slow()("emoji-sheet: only partial file is read for APNG", async t => { + await runSingleTest(t, "https://ezgif.com/images/format-demo/butterfly.png", 2438998) +}) + +slow()("emoji-sheet: only partial file is read for GIF", async t => { + await runSingleTest(t, "https://ezgif.com/images/format-demo/butterfly.gif", 781223) +}) diff --git a/package-lock.json b/package-lock.json index 054e701..5bc10cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "minimist": "^1.2.8", "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0", "node-fetch": "^2.6.7", - "pngjs": "^7.0.0", + "pngjs": "github:cloudrac3r/pngjs#v7.0.1", "prettier-bytes": "^1.0.4", "sharp": "^0.32.6", "snowtransfer": "^0.9.0", @@ -2403,9 +2403,9 @@ } }, "node_modules/pngjs": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", - "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "version": "7.0.1", + "resolved": "git+ssh://git@github.com/cloudrac3r/pngjs.git#55ad02b641a4a4de5cbe00329e8bca600b5bfa3b", + "license": "MIT", "engines": { "node": ">=14.19.0" } diff --git a/package.json b/package.json index 58a8674..c9d3910 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "minimist": "^1.2.8", "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0", "node-fetch": "^2.6.7", - "pngjs": "^7.0.0", + "pngjs": "github:cloudrac3r/pngjs#v7.0.1", "prettier-bytes": "^1.0.4", "sharp": "^0.32.6", "snowtransfer": "^0.9.0", diff --git a/test/test.js b/test/test.js index 6c912c8..8e7f193 100644 --- a/test/test.js +++ b/test/test.js @@ -69,4 +69,5 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../d2m/converters/user-to-mxid.test") require("../m2d/converters/event-to-message.test") require("../m2d/converters/utils.test") + require("../m2d/converters/emoji-sheet.test") })() From 64671519bd10d151b53584a8383950aab017f1c2 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 1 Feb 2024 21:43:10 +1300 Subject: [PATCH 048/431] PK: Fix saving proxy values to DB --- d2m/actions/register-pk-user.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/d2m/actions/register-pk-user.js b/d2m/actions/register-pk-user.js index 003453d..af3f8c9 100644 --- a/d2m/actions/register-pk-user.js +++ b/d2m/actions/register-pk-user.js @@ -128,7 +128,7 @@ async function memberToStateContent(pkMessage, author) { async function syncUser(author, pkMessage, roomID) { const mxid = await ensureSimJoined(pkMessage, roomID) // Update the sim_proxy table, so mentions can look up the original sender later - db.prepare("INSERT OR IGNORE INTO sim_proxy (user_id, proxy_owner_id) VALUES (?, ?)").run(pkMessage.member.id, pkMessage.sender) + db.prepare("INSERT OR IGNORE INTO sim_proxy (user_id, proxy_owner_id) VALUES (?, ?)").run(pkMessage.member.uuid, pkMessage.sender) // Sync the member state const content = await memberToStateContent(pkMessage, author) const currentHash = registerUser._hashProfileContent(content) From 98477dc0f626a5e5407d9122cec64e0f4b790112 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 1 Feb 2024 22:22:48 +1300 Subject: [PATCH 049/431] PK mentions now include member name --- d2m/actions/register-pk-user.js | 2 +- db/migrations/0010-add-sim-proxy.sql | 1 + db/orm-defs.d.ts | 1 + m2d/converters/event-to-message.js | 18 +++++++++++++----- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/d2m/actions/register-pk-user.js b/d2m/actions/register-pk-user.js index af3f8c9..ca47b7c 100644 --- a/d2m/actions/register-pk-user.js +++ b/d2m/actions/register-pk-user.js @@ -128,7 +128,7 @@ async function memberToStateContent(pkMessage, author) { async function syncUser(author, pkMessage, roomID) { const mxid = await ensureSimJoined(pkMessage, roomID) // Update the sim_proxy table, so mentions can look up the original sender later - db.prepare("INSERT OR IGNORE INTO sim_proxy (user_id, proxy_owner_id) VALUES (?, ?)").run(pkMessage.member.uuid, pkMessage.sender) + db.prepare("INSERT OR IGNORE INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES (?, ?, ?)").run(pkMessage.member.uuid, pkMessage.sender, author.username) // Sync the member state const content = await memberToStateContent(pkMessage, author) const currentHash = registerUser._hashProfileContent(content) diff --git a/db/migrations/0010-add-sim-proxy.sql b/db/migrations/0010-add-sim-proxy.sql index d7dd0a1..159bdff 100644 --- a/db/migrations/0010-add-sim-proxy.sql +++ b/db/migrations/0010-add-sim-proxy.sql @@ -1,5 +1,6 @@ CREATE TABLE IF NOT EXISTS sim_proxy ( user_id TEXT NOT NULL, proxy_owner_id TEXT NOT NULL, + displayname TEXT NOT NULL, PRIMARY KEY(user_id) ) WITHOUT ROWID; diff --git a/db/orm-defs.d.ts b/db/orm-defs.d.ts index 35fea51..622e1a0 100644 --- a/db/orm-defs.d.ts +++ b/db/orm-defs.d.ts @@ -66,6 +66,7 @@ export type Models = { sim_proxy: { user_id: string proxy_owner_id: string + displayname: string } webhook: { diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index a48fa75..e48b5f3 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -32,7 +32,7 @@ const markdownEscapes = [ [/^>/g, '\\>'], [/_/g, '\\_'], [/^(\d+)\. /g, '$1\\. '] - ] +] const turndownService = new TurndownService({ hr: "----", @@ -103,7 +103,15 @@ turndownService.addRule("inlineLink", { }, replacement: function (content, node) { - if (node.getAttribute("data-user-id")) return `<@${node.getAttribute("data-user-id")}>` + if (node.getAttribute("data-user-id")) { + const user_id = node.getAttribute("data-user-id") + const row = select("sim_proxy", ["displayname", "proxy_owner_id"], {user_id}).get() + if (row) { + return `**@${row.displayname}** (<@${row.proxy_owner_id}>)` + } else { + return `<@${user_id}>` + } + } if (node.getAttribute("data-message-id")) return `https://discord.com/channels/${node.getAttribute("data-guild-id")}/${node.getAttribute("data-channel-id")}/${node.getAttribute("data-message-id")}` if (node.getAttribute("data-channel-id")) return `<#${node.getAttribute("data-channel-id")}>` const href = node.getAttribute("href") @@ -507,9 +515,9 @@ async function eventToMessage(event, guild, di) { mxid = decodeURIComponent(mxid) if (mxUtils.eventSenderIsFromDiscord(mxid)) { // Handle mention of an OOYE sim user by their mxid - const userID = getUserOrProxyOwnerID(mxid) - if (!userID) return whole - return `${attributeValue} data-user-id="${userID}">` + const id = select("sim", "user_id", {mxid}).pluck().get() + if (!id) return whole + return `${attributeValue} data-user-id="${id}">` } else { // Handle mention of a Matrix user by their mxid // Check if this Matrix user is actually the sim user from another old bridge in the room? From 69922c4a14e4a4dbcd0c603962e87dc1b9850531 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 1 Feb 2024 22:23:08 +1300 Subject: [PATCH 050/431] PK d->m replies are now native Matrix replies --- d2m/converters/message-to-event.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index a620beb..411404c 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -252,6 +252,15 @@ async function messageToEvent(message, guild, options = {}, di) { if (row) { repliedToEventRow = row } + } else if (dUtils.isWebhookMessage(message) && message.embeds[0]?.author?.name?.endsWith("↩️") && message.embeds[0].description?.startsWith("**[Reply to:]")) { + const match = message.embeds[0].description.match(/\/channels\/[0-9]*\/[0-9]*\/([0-9]{2,})/) + if (match) { + const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(match[1]) + if (row) { + message.embeds.shift() + repliedToEventRow = row + } + } } if (repliedToEventRow && repliedToEventRow.source === 0) { // reply was originally from Matrix // Need to figure out who sent that event... From c7fb6fd52eec295914452747f778dfc4fde8d64b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 2 Feb 2024 15:55:02 +1300 Subject: [PATCH 051/431] Improve test coverage --- d2m/converters/message-to-event.js | 9 ++ d2m/converters/message-to-event.pk.test.js | 64 +++++++++++ d2m/converters/message-to-event.test.js | 40 +++++++ db/migrate.js | 1 - m2d/converters/emoji-sheet.test.js | 1 + m2d/converters/event-to-message.js | 90 +++++++++++---- m2d/converters/event-to-message.test.js | 54 ++++++++- test/data.js | 124 +++++++++++++++++++++ test/ooye-test-data.sql | 17 ++- test/test.js | 7 ++ 10 files changed, 374 insertions(+), 33 deletions(-) create mode 100644 d2m/converters/message-to-event.pk.test.js diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 411404c..814fda2 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -257,6 +257,15 @@ async function messageToEvent(message, guild, options = {}, di) { if (match) { const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(match[1]) if (row) { + /* + we generate a partial referenced_message based on what PK provided. we don't need everything, since this will only be used for further message-to-event converting. + the following properties are necessary: + - content: used for generating the reply fallback + */ + // @ts-ignore + message.referenced_message = { + content: message.embeds[0].description.replace(/^.*?\)\*\*\s*/, "") + } message.embeds.shift() repliedToEventRow = row } diff --git a/d2m/converters/message-to-event.pk.test.js b/d2m/converters/message-to-event.pk.test.js new file mode 100644 index 0000000..48984e4 --- /dev/null +++ b/d2m/converters/message-to-event.pk.test.js @@ -0,0 +1,64 @@ +const {test} = require("supertape") +const {messageToEvent} = require("./message-to-event") +const data = require("../../test/data") +const Ty = require("../../types") + +/** + * @param {string} roomID + * @param {string} eventID + * @returns {(roomID: string, eventID: string) => Promise>} + */ +function mockGetEvent(t, roomID_in, eventID_in, outer) { + return async function(roomID, eventID) { + t.equal(roomID, roomID_in) + t.equal(eventID, eventID_in) + return new Promise(resolve => { + setTimeout(() => { + resolve({ + event_id: eventID_in, + room_id: roomID_in, + origin_server_ts: 1680000000000, + unsigned: { + age: 2245, + transaction_id: "$local.whatever" + }, + ...outer + }) + }) + }) + } +} + +test("message2event: pk reply is converted to native matrix reply", async t => { + const events = await messageToEvent(data.pk_message.pk_reply, {}, {}, { + api: { + getEvent: mockGetEvent(t, "!TqlyQmifxGUggEmdBN:cadence.moe", "$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU", { + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "now for my next experiment:" + } + }) + } + }) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": { + user_ids: [ + "@cadence:cadence.moe" + ] + }, + msgtype: "m.text", + body: "> cadence: now for my next experiment:\n\nthis is a reply", + format: "org.matrix.custom.html", + formatted_body: '
In reply to cadence
' + + "now for my next experiment:
" + + "this is a reply", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU" + } + } + }]) +}) diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js index 6b4695d..9f4e4d4 100644 --- a/d2m/converters/message-to-event.test.js +++ b/d2m/converters/message-to-event.test.js @@ -397,6 +397,46 @@ test("message2event: reply with a video", async t => { }]) }) +test("message2event: voice message", async t => { + const events = await messageToEvent(data.message.voice_message) + t.deepEqual(events, [{ + $type: "m.room.message", + body: "voice-message.ogg", + external_url: "https://cdn.discordapp.com/attachments/1099031887500034088/1112476845502365786/voice-message.ogg?ex=65c92d4c&is=65b6b84c&hm=0654bab5027474cbe23875954fa117cf44d8914c144cd151879590fa1baf8b1c&", + filename: "voice-message.ogg", + info: { + duration: 3960.0000381469727, + mimetype: "audio/ogg", + size: 10584, + }, + "m.mentions": {}, + msgtype: "m.audio", + url: "mxc://cadence.moe/MRRPDggXQMYkrUjTpxQbmcxB" + }]) +}) + +test("message2event: misc file", async t => { + const events = await messageToEvent(data.message.misc_file) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + body: "final final final revised draft", + "m.mentions": {} + }, { + $type: "m.room.message", + body: "the.yml", + external_url: "https://cdn.discordapp.com/attachments/122155380120748034/1174514575220158545/the.yml?ex=65cd6270&is=65baed70&hm=8c5f1b571784e3c7f99628492298815884e351ae0dc7c2ae40dd22d97caf27d9&", + filename: "the.yml", + info: { + mimetype: "text/plain; charset=utf-8", + size: 2274 + }, + "m.mentions": {}, + msgtype: "m.file", + url: "mxc://cadence.moe/HnQIYQmmlIKwOQsbFsIGpzPP" + }]) +}) + test("message2event: simple reply in thread to a matrix user's reply", async t => { const events = await messageToEvent(data.message.simple_reply_to_reply_in_thread, data.guild.general, {}, { api: { diff --git a/db/migrate.js b/db/migrate.js index 57b5cbf..7c1faf9 100644 --- a/db/migrate.js +++ b/db/migrate.js @@ -16,7 +16,6 @@ async function migrate(db) { let migrationRan = false for (const filename of files) { - /* c8 ignore next - we can't unit test this, but it's run on every real world bridge startup */ if (progress >= filename) continue console.log(`Applying database migration ${filename}`) if (filename.endsWith(".sql")) { diff --git a/m2d/converters/emoji-sheet.test.js b/m2d/converters/emoji-sheet.test.js index f75fafc..de3a2ab 100644 --- a/m2d/converters/emoji-sheet.test.js +++ b/m2d/converters/emoji-sheet.test.js @@ -39,6 +39,7 @@ async function runSingleTest(t, url, totalSize) { res.body.emit("end") }) t.equal(result.subarray(1, 4).toString("ascii"), "PNG", `result was not a PNG file: ${result.toString("base64")}`) + /* c8 ignore next 5 */ if (meter.bytes < totalSize / 4) { // should download less than 25% of each file t.pass("intentionally read partial file") } else { diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index e48b5f3..2dcb2e8 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -5,6 +5,7 @@ const DiscordTypes = require("discord-api-types/v10") const {Readable} = require("stream") const chunk = require("chunk-text") const TurndownService = require("turndown") +const domino = require("domino") const assert = require("assert").strict const entities = require("entities") @@ -38,7 +39,7 @@ const turndownService = new TurndownService({ hr: "----", headingStyle: "atx", preformattedCode: true, - codeBlockStyle: "fenced", + codeBlockStyle: "fenced" }) /** @@ -339,6 +340,33 @@ async function handleRoomOrMessageLinks(input, di) { return input } +/** + * @param {string} content + * @param {DiscordTypes.APIGuild} guild + * @param {{api: import("../../matrix/api"), snow: import("snowtransfer").SnowTransfer, fetch: import("node-fetch")["default"]}} di + */ +async function checkWrittenMentions(content, guild, di) { + let writtenMentionMatch = content.match(/(?:^|[^"[<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // /d flag for indices requires node.js 16+ + if (writtenMentionMatch) { + const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]}) + if (results[0]) { + assert(results[0].user) + return { + // @ts-ignore - typescript doesn't know about indices yet + content: content.slice(0, writtenMentionMatch.indices[1][0]-1) + `<@${results[0].user.id}>` + content.slice(writtenMentionMatch.indices[1][1]), + ensureJoined: results[0].user + } + } + } +} + +const attachmentEmojis = new Map([ + ["m.image", "🖼️"], + ["m.video", "🎞️"], + ["m.audio", "🎶"], + ["m.file", "📄"] +]) + /** * @param {Ty.Event.Outer_M_Room_Message | Ty.Event.Outer_M_Room_Message_File | Ty.Event.Outer_M_Sticker | Ty.Event.Outer_M_Room_Message_Encrypted_File} event * @param {import("discord-api-types/v10").APIGuild} guild @@ -380,12 +408,10 @@ async function eventToMessage(event, guild, di) { // Handling edits. If the edit was an edit of a reply, edits do not include the reply reference, so we need to fetch up to 2 more events. // this event ---is an edit of--> original event ---is a reply to--> past event await (async () => { - if (!event.content["m.new_content"]) return + // Check if there is an edit const relatesTo = event.content["m.relates_to"] - if (!relatesTo) return + if (!event.content["m.new_content"] || !relatesTo || relatesTo.rel_type !== "m.replace") return // Check if we have a pointer to what was edited - const relType = relatesTo.rel_type - if (relType !== "m.replace") return const originalEventId = relatesTo.event_id if (!originalEventId) return messageIDsToEdit = select("event_message", "message_id", {event_id: originalEventId}, "ORDER BY part").pluck().all() @@ -480,12 +506,7 @@ async function eventToMessage(event, guild, di) { repliedToEvent.content = repliedToEvent.content["m.new_content"] } let contentPreview - const fileReplyContentAlternative = - ( repliedToEvent.content.msgtype === "m.image" ? "🖼️" - : repliedToEvent.content.msgtype === "m.video" ? "🎞️" - : repliedToEvent.content.msgtype === "m.audio" ? "🎶" - : repliedToEvent.content.msgtype === "m.file" ? "📄" - : null) + const fileReplyContentAlternative = attachmentEmojis.get(repliedToEvent.content.msgtype) if (fileReplyContentAlternative) { contentPreview = " " + fileReplyContentAlternative } else { @@ -574,8 +595,35 @@ async function eventToMessage(event, guild, di) { last = match.index } + // Handling written @mentions: we need to look for candidate Discord members to join to the room + // This shouldn't apply to code blocks, links, or inside attributes. So editing the HTML tree instead of regular expressions is a sensible choice here. + // We're using the domino parser because Turndown uses the same and can reuse this tree. + const doc = domino.createDocument( + // DOM parsers arrange elements in the and . Wrapping in a custom element ensures elements are reliably arranged in a single element. + '' + input + '' + ); + const root = doc.getElementById("turndown-root"); + async function forEachNode(node) { + for (; node; node = node.nextSibling) { + if (node.nodeType === 3 && node.nodeValue.includes("@")) { + const result = await checkWrittenMentions(node.nodeValue, guild, di) + if (result) { + node.nodeValue = result.content + ensureJoined.push(result.ensureJoined) + } + } + if (node.nodeType === 1 && ["CODE", "PRE", "A"].includes(node.tagName)) { + // don't recurse into code or links + } else { + // do recurse into everything else + await forEachNode(node.firstChild) + } + } + } + await forEachNode(root) + // @ts-ignore bad type from turndown - content = turndownService.turndown(input) + content = turndownService.turndown(root) // It's designed for commonmark, we need to replace the space-space-newline with just newline content = content.replace(/ \n/g, "\n") @@ -592,6 +640,12 @@ async function eventToMessage(event, guild, di) { content = await handleRoomOrMessageLinks(content, di) + const result = await checkWrittenMentions(content, guild, di) + if (result) { + content = result.content + ensureJoined.push(result.ensureJoined) + } + // Markdown needs to be escaped, though take care not to escape the middle of links // @ts-ignore bad type from turndown content = turndownService.escape(content) @@ -640,18 +694,6 @@ async function eventToMessage(event, guild, di) { content = displayNameRunoff + replyLine + content - // Handling written @mentions: we need to look for candidate Discord members to join to the room - let writtenMentionMatch = content.match(/(?:^|[^"[<>/A-Za-z0-9])@([A-Za-z][A-Za-z0-9._\[\]\(\)-]+):?/d) // /d flag for indices requires node.js 16+ - if (writtenMentionMatch) { - const results = await di.snow.guild.searchGuildMembers(guild.id, {query: writtenMentionMatch[1]}) - if (results[0]) { - assert(results[0].user) - // @ts-ignore - typescript doesn't know about indices yet - content = content.slice(0, writtenMentionMatch.indices[1][0]-1) + `<@${results[0].user.id}>` + content.slice(writtenMentionMatch.indices[1][1]) - ensureJoined.push(results[0].user) - } - } - // Split into 2000 character chunks const chunks = chunk(content, 2000) messages = messages.concat(chunks.map(content => ({ diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 378e9f0..65893fe 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -1924,7 +1924,7 @@ test("event2message: mentioning PK discord users works", async t => { messagesToEdit: [], messagesToSend: [{ username: "cadence [they]", - content: "I'm just <@196188877885538304> testing mentions", + content: "I'm just **@Azalea &flwr; 🌺** (<@196188877885538304>) testing mentions", avatar_url: undefined }] } @@ -2845,7 +2845,7 @@ test("event2message: unknown emojis in the middle are linked", async t => { ) }) -test("event2message: guessed @mentions may join members to mention", async t => { +test("event2message: guessed @mentions in plaintext may join members to mention", async t => { let called = 0 const subtext = { user: { @@ -2893,6 +2893,56 @@ test("event2message: guessed @mentions may join members to mention", async t => t.equal(called, 1, "searchGuildMembers should be called once") }) +test("event2message: guessed @mentions in formatted body may join members to mention", async t => { + let called = 0 + const subtext = { + user: { + id: "321876634777218072", + username: "subtextual", + global_name: "subtext", + discriminator: "0" + } + } + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "wrong body", + format: "org.matrix.custom.html", + formatted_body: "HEY @SUBTEXT, WHAT FOOD WOULD YOU LIKE TO ORDER??" + }, + event_id: "$u5gSwSzv_ZQS3eM00mnTBCor8nx_A_AwuQz7e59PZk8", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }, { + id: "112760669178241024" + }, { + snow: { + guild: { + async searchGuildMembers(guildID, options) { + called++ + t.equal(guildID, "112760669178241024") + t.deepEqual(options, {query: "SUBTEXT"}) + return [subtext] + } + } + } + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "**_HEY <@321876634777218072>, WHAT FOOD WOULD YOU LIKE TO ORDER??_**", + avatar_url: undefined + }], + ensureJoined: [subtext.user] + } + ) + t.equal(called, 1, "searchGuildMembers should be called once") +}) + test("event2message: guessed @mentions work with other matrix bridge old users", async t => { t.deepEqual( await eventToMessage({ diff --git a/test/data.js b/test/data.js index 87ac510..9d86944 100644 --- a/test/data.js +++ b/test/data.js @@ -1340,6 +1340,89 @@ module.exports = { components: [] } }, + voice_message: { + id: "1112476845783388160", + type: 0, + content: "", + channel_id: "1099031887500034088", + author: { + id: "113340068197859328", + username: "kumaccino", + avatar: "b48302623a12bc7c59a71328f72ccb39", + discriminator: "0", + public_flags: 128, + premium_type: 0, + flags: 128, + banner: null, + accent_color: null, + global_name: "kumaccino", + avatar_decoration_data: null, + banner_color: null + }, + attachments: [ + { + id: "1112476845502365786", + filename: "voice-message.ogg", + size: 10584, + url: "https://cdn.discordapp.com/attachments/1099031887500034088/1112476845502365786/voice-message.ogg?ex=65c92d4c&is=65b6b84c&hm=0654bab5027474cbe23875954fa117cf44d8914c144cd151879590fa1baf8b1c&", + proxy_url: "https://media.discordapp.net/attachments/1099031887500034088/1112476845502365786/voice-message.ogg?ex=65c92d4c&is=65b6b84c&hm=0654bab5027474cbe23875954fa117cf44d8914c144cd151879590fa1baf8b1c&", + duration_secs: 3.9600000381469727, + waveform: "AAgXAAwAPBsCAAAAInEDFwAAAAAbMwATEBAAAAAAAAAAAAAAAA==", + content_type: "audio/ogg" + } + ], + embeds: [], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2023-05-28T20:25:48.855000+00:00", + edited_timestamp: null, + flags: 8192, + components: [] + }, + misc_file: { + id: "1174514575819931718", + type: 0, + content: "final final final revised draft", + channel_id: "122155380120748034", + author: { + id: "142843483923677184", + username: "huck", + avatar: "a_1c7fda09a242d714570b4c828ef07504", + discriminator: "0", + public_flags: 512, + premium_type: 2, + flags: 512, + banner: null, + accent_color: null, + global_name: null, + avatar_decoration_data: null, + banner_color: null + }, + attachments: [ + { + id: "1174514575220158545", + filename: "the.yml", + size: 2274, + url: "https://cdn.discordapp.com/attachments/122155380120748034/1174514575220158545/the.yml?ex=65cd6270&is=65baed70&hm=8c5f1b571784e3c7f99628492298815884e351ae0dc7c2ae40dd22d97caf27d9&", + proxy_url: "https://media.discordapp.net/attachments/122155380120748034/1174514575220158545/the.yml?ex=65cd6270&is=65baed70&hm=8c5f1b571784e3c7f99628492298815884e351ae0dc7c2ae40dd22d97caf27d9&", + content_type: "text/plain; charset=utf-8", + content_scan_version: 0 + } + ], + embeds: [], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2023-11-16T01:01:36.301000+00:00", + edited_timestamp: null, + flags: 0, + components: [] + }, simple_reply_to_reply_in_thread: { type: 19, tts: false, @@ -1681,6 +1764,47 @@ module.exports = { components: [] } }, + pk_message: { + pk_reply: { + id: "1202543812644306965", + type: 0, + content: "this is a reply", + channel_id: "1160894080998461480", + author: { + id: "1195662438662680720", + username: "special name", + avatar: "6b44a106659e78a2550474c61889194d", + discriminator: "0000", + public_flags: 0, + flags: 0, + bot: true, + global_name: null + }, + attachments: [], + embeds: [ + { + type: "rich", + description: "**[Reply to:](https://discord.com/channels/1160893336324931584/1160894080998461480/1202543413652881428)** now for my next experiment:", + author: { + name: "cadence [they] ↩️", + icon_url: "https://cdn.discordapp.com/avatars/1162510387057545227/af0ead3b92cf6e448fdad80b4e7fc9e5.png", + proxy_icon_url: "https://images-ext-1.discordapp.net/external/wWslraV-s-bLDwphL64YxeDm30M7PIhQQy0EQa8jpDc/https/cdn.discordapp.com/avatars/1162510387057545227/af0ead3b92cf6e448fdad80b4e7fc9e5.png" + } + } + ], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2024-02-01T09:19:47.118000+00:00", + edited_timestamp: null, + flags: 0, + components: [], + application_id: "466378653216014359", + webhook_id: "1195662438662680720" + } + }, message_with_embeds: { nothing_but_a_field: { guild_id: "497159726455455754", diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index f1b720f..f455d9e 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -12,7 +12,8 @@ INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent, custom ('297272183716052993', '!rEOspnYqdOalaIFniV:cadence.moe', 'general', NULL, NULL, NULL), ('122155380120748034', '!cqeGDbPiMFAhLsqqqq:cadence.moe', 'cadences-mind', 'coding', NULL, NULL), ('176333891320283136', '!qzDBLKlildpzrrOnFZ:cadence.moe', '🌈丨davids-horse_she-took-the-kids', 'wonderland', NULL, 'mxc://cadence.moe/EVvrSkKIRONHjtRJsMLmHWLS'), -('489237891895768942', '!tnedrGVYKFNUdnegvf:tchncs.de', 'ex-room-doesnt-exist-any-more', NULL, NULL, NULL); +('489237891895768942', '!tnedrGVYKFNUdnegvf:tchncs.de', 'ex-room-doesnt-exist-any-more', NULL, NULL, NULL), +('1160894080998461480', '!TqlyQmifxGUggEmdBN:cadence.moe', 'ooyexperiment', NULL, NULL, NULL); INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES ('0', 'bot', '_ooye_bot', '@_ooye_bot:cadence.moe'), @@ -24,8 +25,8 @@ INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES ('1109360903096369153', 'amanda', '_ooye_amanda', '@_ooye_amanda:cadence.moe'), ('43d378d5-1183-47dc-ab3c-d14e21c3fe58', '_pk_zoego', '_ooye__pk_zoego', '@_ooye__pk_zoego:cadence.moe'); -INSERT INTO sim_proxy (user_id, proxy_owner_id) VALUES -('43d378d5-1183-47dc-ab3c-d14e21c3fe58', '196188877885538304'); +INSERT INTO sim_proxy (user_id, proxy_owner_id, displayname) VALUES +('43d378d5-1183-47dc-ab3c-d14e21c3fe58', '196188877885538304', 'Azalea &flwr; 🌺'); INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES ('@_ooye_bojack_horseman:cadence.moe', '!hYnGGlPHlbujVVfktC:cadence.moe', NULL); @@ -48,7 +49,8 @@ INSERT INTO message_channel (message_id, channel_id) VALUES ('1162005526675193909', '1162005314908999790'), ('1162625810109317170', '497161350934560778'), ('1158842413025071135', '176333891320283136'), -('1197612733600895076', '112760669178241024'); +('1197612733600895076', '112760669178241024'), +('1202543413652881428', '1160894080998461480'); INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES ('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 0, 1), @@ -74,7 +76,8 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part ('$0wEdIP8fhTq-P68xwo_gyUw-Zv0KA2aS2tfhdFSrLZc', 'm.room.message', 'm.text', '1162625810109317170', 1, 1, 1), ('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 0, 1), ('$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo', 'm.room.message', 'm.text', '1158842413025071135', 0, 0, 1), -('$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw', 'm.room.message', 'm.text', '1197612733600895076', 0, 0, 1); +('$7tJoMw1h44n2gxgLUE1T_YinGrLbK0x-TDY1z6M7GBw', 'm.room.message', 'm.text', '1197612733600895076', 0, 0, 1), +('$NB6nPgO2tfXyIwwDSF0Ga0BUrsgX1S-0Xl-jAvI8ucU', 'm.room.message', 'm.text', '1202543413652881428', 0, 0, 0); INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'), @@ -92,7 +95,9 @@ INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/emojis/1125827250609201255.png', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP'), ('https://cdn.discordapp.com/avatars/320067006521147393/5fc4ad85c1ea876709e9a7d3374a78a1.png?size=1024', 'mxc://cadence.moe/JPzSmALLirnIprlSMKohSSoX'), ('https://cdn.discordapp.com/emojis/288858540888686602.png', 'mxc://cadence.moe/mwZaCtRGAQQyOItagDeCocEO'), -('https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4', 'mxc://cadence.moe/kMqLycqMURhVpwleWkmASpnU'); +('https://cdn.discordapp.com/attachments/112760669178241024/1197621094786531358/Ins_1960637570.mp4', 'mxc://cadence.moe/kMqLycqMURhVpwleWkmASpnU'), +('https://cdn.discordapp.com/attachments/1099031887500034088/1112476845502365786/voice-message.ogg', 'mxc://cadence.moe/MRRPDggXQMYkrUjTpxQbmcxB'), +('https://cdn.discordapp.com/attachments/122155380120748034/1174514575220158545/the.yml', 'mxc://cadence.moe/HnQIYQmmlIKwOQsbFsIGpzPP'); INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES ('230201364309868544', 'hippo', 0, 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'), diff --git a/test/test.js b/test/test.js index 8e7f193..95daf5b 100644 --- a/test/test.js +++ b/test/test.js @@ -48,6 +48,12 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not t.pass("it did not throw an error") }) await p + + test("migrate: migration works the second time", async t => { + await migrate.migrate(db) + t.pass("it did not throw an error") + }) + db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8")) require("../db/orm.test") require("../discord/utils.test") @@ -63,6 +69,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../d2m/converters/lottie.test") require("../d2m/converters/message-to-event.test") require("../d2m/converters/message-to-event.embeds.test") + require("../d2m/converters/message-to-event.pk.test") require("../d2m/converters/pins-to-list.test") require("../d2m/converters/remove-reaction.test") require("../d2m/converters/thread-to-announcement.test") From 0e701b2d54b970a20f0d0608364d10e74512bdaa Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 7 Feb 2024 14:52:12 +1300 Subject: [PATCH 052/431] Fix speedbump in threads --- d2m/actions/delete-message.js | 4 ++-- d2m/actions/speedbump.js | 19 ++++++++++++++++++- d2m/event-dispatcher.js | 15 ++++----------- readme.md | 7 ++++--- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/d2m/actions/delete-message.js b/d2m/actions/delete-message.js index 4386ae5..440e123 100644 --- a/d2m/actions/delete-message.js +++ b/d2m/actions/delete-message.js @@ -11,7 +11,7 @@ const speedbump = sync.require("./speedbump") * @param {import("discord-api-types/v10").GatewayMessageDeleteDispatchData} data */ async function deleteMessage(data) { - const row = select("channel_room", ["room_id", "speedbump_checked"], {channel_id: data.channel_id}).get() + const row = select("channel_room", ["room_id", "speedbump_checked", "thread_parent"], {channel_id: data.channel_id}).get() if (!row) return const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all() @@ -22,7 +22,7 @@ async function deleteMessage(data) { await api.redactEvent(row.room_id, eventID) } - speedbump.updateCache(data.channel_id, row.speedbump_checked) + await speedbump.updateCache(row.thread_parent || data.channel_id, row.speedbump_checked) } /** diff --git a/d2m/actions/speedbump.js b/d2m/actions/speedbump.js index f49a378..e782ae0 100644 --- a/d2m/actions/speedbump.js +++ b/d2m/actions/speedbump.js @@ -2,7 +2,7 @@ const DiscordTypes = require("discord-api-types/v10") const passthrough = require("../../passthrough") -const {discord, db} = passthrough +const {discord, select, db} = passthrough const SPEEDBUMP_SPEED = 4000 // 4 seconds delay const SPEEDBUMP_UPDATE_FREQUENCY = 2 * 60 * 60 // 2 hours @@ -33,6 +33,7 @@ const bumping = new Set() /** * Slow down a message. After it passes the speedbump, return whether it's okay or if it's been deleted. * @param {string} messageID + * @returns whether it was deleted */ async function doSpeedbump(messageID) { bumping.add(messageID) @@ -40,6 +41,21 @@ async function doSpeedbump(messageID) { return !bumping.delete(messageID) } +/** + * Check whether to slow down a message, and do it. After it passes the speedbump, return whether it's okay or if it's been deleted. + * @param {string} channelID + * @param {string} messageID + * @returns whether it was deleted, and data about the channel's (not thread's) speedbump + */ +async function maybeDoSpeedbump(channelID, messageID) { + let row = select("channel_room", ["thread_parent", "speedbump_id", "speedbump_webhook_id"], {channel_id: channelID}).get() + if (row?.thread_parent) row = select("channel_room", ["thread_parent", "speedbump_id", "speedbump_webhook_id"], {channel_id: row.thread_parent}).get() // webhooks belong to the channel, not the thread + if (!row) return {affected: false, row: null}// not affected, no speedbump + // Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from. + const affected = await doSpeedbump(messageID) + return {affected, row} // maybe affected, and there is a speedbump +} + /** * @param {string} messageID */ @@ -49,4 +65,5 @@ function onMessageDelete(messageID) { module.exports.updateCache = updateCache module.exports.doSpeedbump = doSpeedbump +module.exports.maybeDoSpeedbump = maybeDoSpeedbump module.exports.onMessageDelete = onMessageDelete diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index c630bfb..52bed72 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -246,11 +246,8 @@ module.exports = { if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. } - const row = select("channel_room", ["speedbump_id", "speedbump_webhook_id"], {channel_id: message.channel_id}).get() - if (row && row.speedbump_id) { - const affected = await speedbump.doSpeedbump(message.id) - if (affected) return - } + const {affected, row} = await speedbump.maybeDoSpeedbump(message.channel_id, message.id) + if (affected) return // @ts-ignore await sendMessage.sendMessage(message, guild, row), @@ -267,12 +264,8 @@ module.exports = { if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. } - const row = select("channel_room", ["speedbump_id", "speedbump_webhook_id"], {channel_id: data.channel_id}).get() - if (row) { - // Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from. - const affected = await speedbump.doSpeedbump(data.id) - if (affected) return - } + const {affected, row} = await speedbump.maybeDoSpeedbump(data.channel_id, data.id) + if (affected) return // Based on looking at data they've sent me over the gateway, this is the best way to check for meaningful changes. // If the message content is a string then it includes all interesting fields and is meaningful. diff --git a/readme.md b/readme.md index 05d8f9c..49c2e66 100644 --- a/readme.md +++ b/readme.md @@ -2,9 +2,9 @@ -Modern Matrix-to-Discord appservice bridge. +Modern Matrix-to-Discord appservice bridge, created by [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe) -Created by [@cadence:cadence.moe](https://matrix.to/#/@cadence:cadence.moe) // Discuss in [#out-of-your-element:cadence.moe](https://matrix.to/#/#out-of-your-element:cadence.moe) +[![Releases](https://img.shields.io/gitea/v/release/cadence/out-of-your-element?gitea_url=https%3A%2F%2Fgitdab.com&style=plastic&color=green)](https://gitdab.com/cadence/out-of-your-element/releases) [![Discuss on Matrix](https://img.shields.io/badge/discuss-%23out--of--your--element-white?style=plastic)](https://matrix.to/#/#out-of-your-element:cadence.moe) ## Docs @@ -76,7 +76,8 @@ Follow these steps: 1. [Get Node.js version 18 or later](https://nodejs.org/en/download/releases) (the version is required by the better-sqlite3 and matrix-appservice dependencies) -1. Clone this repo and checkout a specific tag. (Development happens on main. Stabler versions are tagged.) +1. Clone this repo and checkout a specific tag. (Development happens on main. Stable versions are tagged.) + * The latest release tag is ![](https://img.shields.io/gitea/v/release/cadence/out-of-your-element?gitea_url=https%3A%2F%2Fgitdab.com&style=flat-square&label=%20&color=black). 1. Install dependencies: `npm install --save-dev` (omit --save-dev if you will not run the automated tests) From cfc89c40f96552e98bfb9e1dc1e2ba77aae88cad Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 7 Feb 2024 16:53:50 +1300 Subject: [PATCH 053/431] d->m: test: guessed @mentions don't change in code --- m2d/converters/event-to-message.test.js | 39 +++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 65893fe..f116f8c 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -2943,6 +2943,45 @@ test("event2message: guessed @mentions in formatted body may join members to men t.equal(called, 1, "searchGuildMembers should be called once") }) +test("event2message: guessed @mentions feature will not activate on links or code", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "wrong body @subtext wrong body", + format: "org.matrix.custom.html", + formatted_body: 'in link view timeline' + + ' in autolink https://example.com/social/@subtext' + + ' in pre-code 
@subtext
' + }, + event_id: "$u5gSwSzv_ZQS3eM00mnTBCor8nx_A_AwuQz7e59PZk8", + room_id: "!kLRqKKUQXcibIMtOpl:cadence.moe" + }, {}, { + snow: { + guild: { + /* c8 ignore next 4 */ + async searchGuildMembers() { + t.fail("the feature activated when it wasn't supposed to") + return [] + } + } + } + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "in link [view timeline](https://example.com/social/@subtext) in autolink https://example.com/social/@subtext in pre-code```\n@subtext\n```", + avatar_url: undefined + }], + ensureJoined: [] + } + ) +}) + test("event2message: guessed @mentions work with other matrix bridge old users", async t => { t.deepEqual( await eventToMessage({ From 3fb2c983e09ce1a2c7412c6e4cbf828daf4f15fa Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 7 Feb 2024 23:00:41 +1300 Subject: [PATCH 054/431] Fix pngjs install --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index e4a0ab2..14318ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,7 @@ "minimist": "^1.2.8", "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0", "node-fetch": "^2.6.7", - "pngjs": "github:cloudrac3r/pngjs#v7.0.1", + "pngjs": "github:cloudrac3r/pngjs#v7.0.2", "prettier-bytes": "^1.0.4", "sharp": "^0.32.6", "snowtransfer": "^0.9.0", @@ -2415,8 +2415,8 @@ } }, "node_modules/pngjs": { - "version": "7.0.1", - "resolved": "git+ssh://git@github.com/cloudrac3r/pngjs.git#55ad02b641a4a4de5cbe00329e8bca600b5bfa3b", + "version": "7.0.2", + "resolved": "git+ssh://git@github.com/cloudrac3r/pngjs.git#0295be509ed56dcf2f1d11b3af0b3108ad699dfe", "license": "MIT", "engines": { "node": ">=14.19.0" diff --git a/package.json b/package.json index 33d3064..63c699d 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "minimist": "^1.2.8", "mixin-deep": "github:cloudrac3r/mixin-deep#v3.0.0", "node-fetch": "^2.6.7", - "pngjs": "github:cloudrac3r/pngjs#v7.0.1", + "pngjs": "github:cloudrac3r/pngjs#v7.0.2", "prettier-bytes": "^1.0.4", "sharp": "^0.32.6", "snowtransfer": "^0.9.0", From 30afaa1e17a165f9b492589d9f5cbcc833b3be0a Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 9 Feb 2024 17:29:05 +1300 Subject: [PATCH 055/431] Add getViaServers function --- m2d/converters/utils.js | 66 +++++++++++++++++++++++++++++ m2d/converters/utils.test.js | 80 +++++++++++++++++++++++++++++++++++- matrix/api.js | 2 +- 3 files changed, 146 insertions(+), 2 deletions(-) diff --git a/m2d/converters/utils.js b/m2d/converters/utils.js index 8a83a07..5707aec 100644 --- a/m2d/converters/utils.js +++ b/m2d/converters/utils.js @@ -127,8 +127,74 @@ class MatrixStringBuilder { } } +/** + * Room IDs are not routable on their own. Room permalinks need a list of servers to try. The client is responsible for coming up with a list of servers. + * https://spec.matrix.org/v1.9/appendices/#routing + * https://gitdab.com/cadence/out-of-your-element/issues/11 + * @param {string} roomID + * @param {{[K in "getStateEvent" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api + */ +async function getViaServers(roomID, api) { + const candidates = [] + const {joined} = await api.getJoinedMembers(roomID) + // Candidate 0: The bot's own server name + candidates.push(reg.ooye.server_name) + // Candidate 1: Highest joined non-sim non-bot power level user in the room + // https://github.com/matrix-org/matrix-react-sdk/blob/552c65db98b59406fb49562e537a2721c8505517/src/utils/permalinks/Permalinks.ts#L172 + try { + /** @type {{users?: {[mxid: string]: number}}} */ + const powerLevels = await api.getStateEvent(roomID, "m.room.power_levels", "") + if (powerLevels.users) { + const sorted = Object.entries(powerLevels.users).sort((a, b) => b[1] - a[1]) // Highest... + for (const power of sorted) { + const mxid = power[0] + if (!(mxid in joined)) continue // joined... + if (userRegex.some(r => mxid.match(r))) continue // non-sim non-bot... + const match = mxid.match(/:(.*)/) + assert(match) + if (!candidates.includes(match[1])) { + candidates.push(match[1]) + break + } + } + } + } catch (e) { + // power levels event not found + } + // Candidates 2-3: Most popular servers in the room + /** @type {Map} */ + const servers = new Map() + // We can get the most popular servers if we know the members, so let's process those... + Object.keys(joined) + .filter(mxid => !mxid.startsWith("@_")) // Quick check + .filter(mxid => !userRegex.some(r => mxid.match(r))) // Full check + .slice(0, 1000) // Just sample the first thousand real members + .map(mxid => { + const match = mxid.match(/:(.*)/) + assert(match) + return match[1] + }) + .filter(server => !server.match(/([a-f0-9:]+:+)+[a-f0-9]+/)) // No IPv6 servers + .filter(server => !server.match(/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)) // No IPv4 servers + // I don't care enough to check ACLs + .forEach(server => { + const existing = servers.get(server) + if (!existing) servers.set(server, 1) + else servers.set(server, existing + 1) + }) + const serverList = [...servers.entries()].sort((a, b) => b[1] - a[1]) + for (const server of serverList) { + if (!candidates.includes(server[0])) { + candidates.push(server[0]) + if (candidates.length >= 4) break // Can have at most 4 candidate via servers + } + } + return candidates +} + module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord module.exports.getPublicUrlForMxc = getPublicUrlForMxc module.exports.getEventIDHash = getEventIDHash module.exports.MatrixStringBuilder = MatrixStringBuilder +module.exports.getViaServers = getViaServers diff --git a/m2d/converters/utils.test.js b/m2d/converters/utils.test.js index 76fd824..3689a87 100644 --- a/m2d/converters/utils.test.js +++ b/m2d/converters/utils.test.js @@ -3,9 +3,22 @@ const e = new Error("Custom error") const {test} = require("supertape") -const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder} = require("./utils") +const {eventSenderIsFromDiscord, getEventIDHash, MatrixStringBuilder, getViaServers} = require("./utils") const util = require("util") +/** @param {string[]} mxids */ +function joinedList(mxids) { + /** @type {{[mxid: string]: {display_name: null, avatar_url: null}}} */ + const joined = {} + for (const mxid of mxids) { + joined[mxid] = { + display_name: null, + avatar_url: null + } + } + return {joined} +} + test("sender type: matrix user", t => { t.notOk(eventSenderIsFromDiscord("@cadence:cadence.moe")) }) @@ -74,3 +87,68 @@ test("MatrixStringBuilder: complete code coverage", t => { formatted_body: "Line 1

Line 2

Line 3

Line 4

" }) }) + +test("getViaServers: returns the server name if the room only has sim users", async t => { + const result = await getViaServers("!baby", { + getStateEvent: async () => ({}), + getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe"]) + }) + t.deepEqual(result, ["cadence.moe"]) +}) + +test("getViaServers: also returns the most popular servers in order", async t => { + const result = await getViaServers("!baby", { + getStateEvent: async () => ({}), + getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid"]) + }) + t.deepEqual(result, ["cadence.moe", "thecollective.invalid", "selfhosted.invalid"]) +}) + +test("getViaServers: does not return IP address servers", async t => { + const result = await getViaServers("!baby", { + getStateEvent: async () => ({}), + getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:45.77.232.172:8443", "@cadence:[::1]:8443", "@cadence:123example.456example.invalid"]) + }) + t.deepEqual(result, ["cadence.moe", "123example.456example.invalid"]) +}) + +test("getViaServers: also returns the highest power level user (100)", async t => { + const result = await getViaServers("!baby", { + getStateEvent: async () => ({ + users: { + "@moderator:tractor.invalid": 50, + "@singleuser:selfhosted.invalid": 100, + "@_ooye_bot:cadence.moe": 100 + } + }), + getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@moderator:tractor.invalid"]) + }) + t.deepEqual(result, ["cadence.moe", "selfhosted.invalid", "thecollective.invalid", "tractor.invalid"]) +}) + +test("getViaServers: also returns the highest power level user (50)", async t => { + const result = await getViaServers("!baby", { + getStateEvent: async () => ({ + users: { + "@moderator:tractor.invalid": 50, + "@_ooye_bot:cadence.moe": 100 + } + }), + getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@hazel:thecollective.invalid", "@june:thecollective.invalid", "@singleuser:selfhosted.invalid"]) + }) + t.deepEqual(result, ["cadence.moe", "tractor.invalid", "thecollective.invalid", "selfhosted.invalid"]) +}) + +test("getViaServers: returns at most 4 results", async t => { + const result = await getViaServers("!baby", { + getStateEvent: async () => ({ + users: { + "@moderator:tractor.invalid": 50, + "@singleuser:selfhosted.invalid": 100, + "@_ooye_bot:cadence.moe": 100 + } + }), + getJoinedMembers: async () => joinedList(["@_ooye_bot:cadence.moe", "@_ooye_hazel:cadence.moe", "@cadence:cadence.moe", "@moderator:tractor.invalid", "@singleuser:selfhosted.invalid", "@hazel:thecollective.invalid", "@cadence:123example.456example.invalid"]) + }) + t.deepEqual(result.length, 4) +}) diff --git a/matrix/api.js b/matrix/api.js index b59d6ef..baa5d96 100644 --- a/matrix/api.js +++ b/matrix/api.js @@ -115,7 +115,7 @@ function getStateEvent(roomID, type, key) { /** * "Any of the AS's users must be in the room. This API is primarily for Application Services and should be faster to respond than /members as it can be implemented more efficiently on the server." * @param {string} roomID - * @returns {Promise<{joined: {[mxid: string]: {avatar_url?: string, display_name?: string}}}>} + * @returns {Promise<{joined: {[mxid: string]: {avatar_url: string?, display_name: string?}}}>} */ function getJoinedMembers(roomID) { return mreq.mreq("GET", `/client/v3/rooms/${roomID}/joined_members`) From 4286829b427e2dd676d523f4044fe668042da69d Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 10 Feb 2024 16:54:38 +1300 Subject: [PATCH 056/431] Update discord libraries --- package-lock.json | 43 ++++++++++++++++++++++--------------------- package.json | 4 ++-- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index 14318ab..9c4f1bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@chriscdn/promise-semaphore": "^2.0.1", "better-sqlite3": "^9.0.0", "chunk-text": "^2.0.1", - "cloudstorm": "^0.9.5", + "cloudstorm": "^0.10.7", "deep-equal": "^2.2.3", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", @@ -28,7 +28,7 @@ "pngjs": "github:cloudrac3r/pngjs#v7.0.2", "prettier-bytes": "^1.0.4", "sharp": "^0.32.6", - "snowtransfer": "^0.9.0", + "snowtransfer": "^0.10.4", "stream-mime-type": "^1.0.2", "try-to-catch": "^3.0.1", "turndown": "^7.1.2", @@ -156,9 +156,9 @@ } }, "node_modules/@fastify/busboy": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.0.0.tgz", - "integrity": "sha512-JUFJad5lv7jxj926GPgymrWQxxjPYuJNiNjNMzqT+HiuP6Vl3dk5xzG+8sTX96np0ZAluvaMzPsjhHZ5rNuNQQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", + "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==", "engines": { "node": ">=14" } @@ -784,11 +784,12 @@ } }, "node_modules/cloudstorm": { - "version": "0.9.5", - "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.9.5.tgz", - "integrity": "sha512-WKaCsTDobR5c3YOmAchIa4QhPyWkUtYP3wNC/h6iE4bXE1DdN432FD3u3cuD3fX1Km9fPgpGBi4m6KYf5GFkJg==", + "version": "0.10.7", + "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.10.7.tgz", + "integrity": "sha512-AgrwjaxSxdMPX8MIbsQYop9g9THRGsu2m6GO8d+QOPoPTErih50jGw3CXgkqyqrpTQadmqg5x+0PyeI/EFYu/w==", "dependencies": { - "snowtransfer": "^0.9.0" + "discord-api-types": "^0.37.69", + "snowtransfer": "^0.10.4" }, "engines": { "node": ">=14.8.0" @@ -1069,9 +1070,9 @@ } }, "node_modules/discord-api-types": { - "version": "0.37.60", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.60.tgz", - "integrity": "sha512-5BELXTsv7becqVHrD81nZrqT4oEyXXWBwbsO/kwDDu6X3u19VV1tYDB5I5vaVAK+c1chcDeheI9zACBLm41LiQ==" + "version": "0.37.69", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.69.tgz", + "integrity": "sha512-c0rHc5YGNIXQkI+V7QwP8y77wxo74ITNeZmMwxtKC/l01aIF/gKBG/U2MKhUt2iaeRH9XwAt9PT3AI9JQVvKVA==" }, "node_modules/discord-markdown": { "version": "2.5.1", @@ -2986,12 +2987,12 @@ } }, "node_modules/snowtransfer": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.9.0.tgz", - "integrity": "sha512-43Q0pvk7ZV8uZwcL/IhEFYKFZj53FOqxr2dVDwduPT87eHOJzfs8aQ+tNDqsjW6OMUBurwR3XZZFEpQ2f/XzXA==", + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.10.4.tgz", + "integrity": "sha512-YKCRS6lZJF59Iz3iJNrqv0MPfvHCxs3Q6KiEOtKA+IhMsffOwdk6K9MkNrGLJWE9hDtObgQ9C8w9NaPrtW+p3A==", "dependencies": { - "discord-api-types": "^0.37.60", - "undici": "^5.26.3" + "discord-api-types": "^0.37.67", + "undici": "^6.5.0" }, "engines": { "node": ">=14.18.0" @@ -3455,14 +3456,14 @@ } }, "node_modules/undici": { - "version": "5.26.3", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.3.tgz", - "integrity": "sha512-H7n2zmKEWgOllKkIUkLvFmsJQj062lSm3uA4EYApG8gLuiOM0/go9bIoC3HVaSnfg4xunowDE2i9p8drkXuvDw==", + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.6.2.tgz", + "integrity": "sha512-vSqvUE5skSxQJ5sztTZ/CdeJb1Wq0Hf44hlYMciqHghvz+K88U0l7D6u1VsndoFgskDcnU+nG3gYmMzJVzd9Qg==", "dependencies": { "@fastify/busboy": "^2.0.0" }, "engines": { - "node": ">=14.0" + "node": ">=18.0" } }, "node_modules/unpipe": { diff --git a/package.json b/package.json index 63c699d..5b22a4f 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "@chriscdn/promise-semaphore": "^2.0.1", "better-sqlite3": "^9.0.0", "chunk-text": "^2.0.1", - "cloudstorm": "^0.9.5", + "cloudstorm": "^0.10.7", "deep-equal": "^2.2.3", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", @@ -34,7 +34,7 @@ "pngjs": "github:cloudrac3r/pngjs#v7.0.2", "prettier-bytes": "^1.0.4", "sharp": "^0.32.6", - "snowtransfer": "^0.9.0", + "snowtransfer": "^0.10.4", "stream-mime-type": "^1.0.2", "try-to-catch": "^3.0.1", "turndown": "^7.1.2", From a9f57fc2528a5e5132d17adee2a30cb75c798825 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 12 Feb 2024 23:07:55 +1300 Subject: [PATCH 057/431] WIP add via parameters --- d2m/actions/speedbump.js | 3 +-- d2m/converters/message-to-event.js | 30 ++++++++++++++++++++++------- d2m/event-dispatcher.js | 2 ++ m2d/converters/utils.js | 24 ++++++++++++++++++++++- package-lock.json | 31 +++++++++++++++--------------- package.json | 2 +- 6 files changed, 65 insertions(+), 27 deletions(-) diff --git a/d2m/actions/speedbump.js b/d2m/actions/speedbump.js index e782ae0..7c3109b 100644 --- a/d2m/actions/speedbump.js +++ b/d2m/actions/speedbump.js @@ -50,8 +50,7 @@ async function doSpeedbump(messageID) { async function maybeDoSpeedbump(channelID, messageID) { let row = select("channel_room", ["thread_parent", "speedbump_id", "speedbump_webhook_id"], {channel_id: channelID}).get() if (row?.thread_parent) row = select("channel_room", ["thread_parent", "speedbump_id", "speedbump_webhook_id"], {channel_id: row.thread_parent}).get() // webhooks belong to the channel, not the thread - if (!row) return {affected: false, row: null}// not affected, no speedbump - // Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from. + if (!row?.speedbump_webhook_id) return {affected: false, row: null} // not affected, no speedbump const affected = await doSpeedbump(messageID) return {affected, row} // maybe affected, and there is a speedbump } diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index 814fda2..1b21be4 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -39,15 +39,15 @@ function getDiscordParseCallbacks(message, guild, useHTML) { return `@${username}:` } }, - /** @param {{id: string, type: "discordChannel"}} node */ + // FIXME: type + /** @param {{id: string, type: "discordChannel", row: any, via: URLSearchParams}} node */ channel: node => { - const row = select("channel_room", ["room_id", "name", "nick"], {channel_id: node.id}).get() - if (!row) { + if (!node.row) { return `#[channel-from-an-unknown-server]` // fallback for when this channel is not bridged } else if (useHTML) { - return `#${row.nick || row.name}` + return `#${node.row.nick || node.row.name}` } else { - return `#${row.nick || row.name}` + return `#${node.row.nick || node.row.name}` } }, /** @param {{animated: boolean, name: string, id: string, type: "discordEmoji"}} node */ @@ -332,12 +332,27 @@ async function messageToEvent(message, guild, options = {}, di) { return emojiToKey.emojiToKey({id, name, animated}) // Register the custom emoji if needed })) - let html = markdown.toHTML(content, { + async function transformParsedVia(parsed) { + for (const node of parsed) { + if (node.type === "discordChannel") { + node.row = select("channel_room", ["room_id", "name", "nick"], {channel_id: node.id}).get() + if (node.row?.room_id) { + node.via = await mxUtils.getViaServersQuery(node.row.room_id, di.api) + } + } + if (Array.isArray(node.content)) { + await transformParsedVia(node.content) + } + } + return parsed + } + + let html = await markdown.toHtmlWithPostParser(content, transformParsedVia, { discordCallback: getDiscordParseCallbacks(message, guild, true), ...customOptions }, customParser, customHtmlOutput) - let body = markdown.toHTML(content, { + let body = await markdown.toHtmlWithPostParser(content, transformParsedVia, { discordCallback: getDiscordParseCallbacks(message, guild, false), discordOnly: true, escapeHTML: false, @@ -347,6 +362,7 @@ async function messageToEvent(message, guild, options = {}, di) { return {body, html} } + // FIXME: What was the scanMentions parameter supposed to activate? It's unused. async function addTextEvent(body, html, msgtype, {scanMentions}) { // Star * prefix for fallback edits if (options.includeEditFallbackStar) { diff --git a/d2m/event-dispatcher.js b/d2m/event-dispatcher.js index 52bed72..e769053 100644 --- a/d2m/event-dispatcher.js +++ b/d2m/event-dispatcher.js @@ -264,6 +264,7 @@ module.exports = { if (row) return // The message was sent by the bridge's own webhook on discord. We don't want to reflect this back, so just drop it. } + // Edits need to go through the speedbump as well. If the message is delayed but the edit isn't, we don't have anything to edit from. const {affected, row} = await speedbump.maybeDoSpeedbump(data.channel_id, data.id) if (affected) return @@ -273,6 +274,7 @@ module.exports = { /** @type {DiscordTypes.GatewayMessageCreateDispatchData} */ // @ts-ignore const message = data + const channel = client.channels.get(message.channel_id) if (!channel || !("guild_id" in channel) || !channel.guild_id) return // Nothing we can do in direct messages. const guild = client.guilds.get(channel.guild_id) diff --git a/m2d/converters/utils.js b/m2d/converters/utils.js index 5707aec..9d7a5e2 100644 --- a/m2d/converters/utils.js +++ b/m2d/converters/utils.js @@ -128,7 +128,9 @@ class MatrixStringBuilder { } /** - * Room IDs are not routable on their own. Room permalinks need a list of servers to try. The client is responsible for coming up with a list of servers. + * Context: Room IDs are not routable on their own. Room permalinks need a list of servers to try. The client is responsible for coming up with a list of servers. + * ASSUMPTION 1: The bridge bot is a member of the target room and can therefore access its power levels and member list for calculation. + * ASSUMPTION 2: Because the bridge bot is a member of the target room, the target room is bridged. * https://spec.matrix.org/v1.9/appendices/#routing * https://gitdab.com/cadence/out-of-your-element/issues/11 * @param {string} roomID @@ -192,9 +194,29 @@ async function getViaServers(roomID, api) { return candidates } +/** + * Context: Room IDs are not routable on their own. Room permalinks need a list of servers to try. The client is responsible for coming up with a list of servers. + * ASSUMPTION 1: The bridge bot is a member of the target room and can therefore access its power levels and member list for calculation. + * ASSUMPTION 2: Because the bridge bot is a member of the target room, the target room is bridged. + * https://spec.matrix.org/v1.9/appendices/#routing + * https://gitdab.com/cadence/out-of-your-element/issues/11 + * @param {string} roomID + * @param {{[K in "getStateEvent" | "getJoinedMembers"]: import("../../matrix/api")[K]}} api + * @returns {Promise} + */ +async function getViaServersQuery(roomID, api) { + const list = await getViaServers(roomID, api) + const qs = new URLSearchParams() + for (const server of list) { + qs.append("via", server) + } + return qs +} + module.exports.BLOCK_ELEMENTS = BLOCK_ELEMENTS module.exports.eventSenderIsFromDiscord = eventSenderIsFromDiscord module.exports.getPublicUrlForMxc = getPublicUrlForMxc module.exports.getEventIDHash = getEventIDHash module.exports.MatrixStringBuilder = MatrixStringBuilder module.exports.getViaServers = getViaServers +module.exports.getViaServersQuery = getViaServersQuery diff --git a/package-lock.json b/package-lock.json index 9c4f1bf..27e04e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "chunk-text": "^2.0.1", "cloudstorm": "^0.10.7", "deep-equal": "^2.2.3", - "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", + "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de44d1666a9115a3763f45ccdcd414c3bf4dbbef", "entities": "^4.5.0", "get-stream": "^6.0.1", "giframe": "github:cloudrac3r/giframe#v0.4.1", @@ -352,14 +352,14 @@ } }, "node_modules/@types/prop-types": { - "version": "15.7.5", - "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", - "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==" + "version": "15.7.11", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", + "integrity": "sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==" }, "node_modules/@types/react": { - "version": "18.2.20", - "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.20.tgz", - "integrity": "sha512-WKNtmsLWJM/3D5mG4U84cysVY31ivmyw85dE84fOCk5Hx78wezB/XEjVPWl2JTZ5FkEeaTJf+VgUAUn3PE7Isw==", + "version": "18.2.55", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.55.tgz", + "integrity": "sha512-Y2Tz5P4yz23brwm2d7jNon39qoAtMMmalOQv6+fEFt1mT+FcM3D841wDpoUvFXhaYenuROCy3FZYqdTjM7qVyA==", "dependencies": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -367,9 +367,9 @@ } }, "node_modules/@types/scheduler": { - "version": "0.16.3", - "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.3.tgz", - "integrity": "sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==" + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz", + "integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==" }, "node_modules/abort-controller": { "version": "3.0.0", @@ -927,9 +927,9 @@ } }, "node_modules/csstype": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", - "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==" + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==" }, "node_modules/data-uri-to-buffer": { "version": "2.0.2", @@ -1075,9 +1075,8 @@ "integrity": "sha512-c0rHc5YGNIXQkI+V7QwP8y77wxo74ITNeZmMwxtKC/l01aIF/gKBG/U2MKhUt2iaeRH9XwAt9PT3AI9JQVvKVA==" }, "node_modules/discord-markdown": { - "version": "2.5.1", - "resolved": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", - "license": "MIT", + "version": "2.6.0", + "resolved": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de44d1666a9115a3763f45ccdcd414c3bf4dbbef", "dependencies": { "simple-markdown": "^0.7.2" } diff --git a/package.json b/package.json index 5b22a4f..5ca5f32 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "chunk-text": "^2.0.1", "cloudstorm": "^0.10.7", "deep-equal": "^2.2.3", - "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", + "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de44d1666a9115a3763f45ccdcd414c3bf4dbbef", "entities": "^4.5.0", "get-stream": "^6.0.1", "giframe": "github:cloudrac3r/giframe#v0.4.1", From 789a90a893d3af38d94baa5c61a68b83e0425f4b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 13 Feb 2024 08:27:58 +1300 Subject: [PATCH 058/431] WIP tests for via parameters --- .../message-to-event.embeds.test.js | 34 ++++++++++++++++--- d2m/converters/message-to-event.test.js | 30 ++++++++++++++-- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/d2m/converters/message-to-event.embeds.test.js b/d2m/converters/message-to-event.embeds.test.js index b339687..96ba53d 100644 --- a/d2m/converters/message-to-event.embeds.test.js +++ b/d2m/converters/message-to-event.embeds.test.js @@ -75,22 +75,48 @@ test("message2event embeds: image embed and attachment", async t => { }) test("message2event embeds: blockquote in embed", async t => { - const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general) + let called = 0 + const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general, {}, { + api: { + async getStateEvent(roomID, type, key) { + called++ + t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe") + t.equal(type, "m.room.power_levels") + t.equal(key, "") + return { + users: { + "@_ooye_bot:cadence.moe": 100 + } + } + }, + async getJoinedMembers(roomID) { + called++ + t.equal(roomID, "!qzDBLKlildpzrrOnFZ:cadence.moe") + return { + joined: { + "@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null}, + "@user:example.invalid": {display_name: null, avatar_url: null} + } + } + } + } + }) t.deepEqual(events, [{ $type: "m.room.message", msgtype: "m.text", body: ":emoji: **4 |** #wonderland", format: "org.matrix.custom.html", - formatted_body: `\":emoji:\" 4 | #wonderland`, + formatted_body: `\":emoji:\" 4 | #wonderland`, "m.mentions": {} }, { $type: "m.room.message", msgtype: "m.notice", - body: "> ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo\n> \n> reply draft\n> > The following is a message composed via consensus of the Stinker Council.\n> > \n> > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n> > \n> > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n> > \n> > There will be no further communication.\n> \n> [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo)", + body: "> ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid\n> \n> reply draft\n> > The following is a message composed via consensus of the Stinker Council.\n> > \n> > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n> > \n> > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n> > \n> > There will be no further communication.\n> \n> [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid)", format: "org.matrix.custom.html", - formatted_body: "

⏺️ minimus

reply draft

The following is a message composed via consensus of the Stinker Council.

For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.

Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.

There will be no further communication.

Go to Message

", + formatted_body: "

⏺️ minimus

reply draft

The following is a message composed via consensus of the Stinker Council.

For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.

Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.

There will be no further communication.

Go to Message

", "m.mentions": {} }]) + t.equal(called, 2) }) test("message2event embeds: crazy html is all escaped", async t => { diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js index 9f4e4d4..9154ac4 100644 --- a/d2m/converters/message-to-event.test.js +++ b/d2m/converters/message-to-event.test.js @@ -62,15 +62,41 @@ test("message2event: simple user mention", async t => { }) test("message2event: simple room mention", async t => { - const events = await messageToEvent(data.message.simple_room_mention, data.guild.general, {}) + let called = 0 + const events = await messageToEvent(data.message.simple_room_mention, data.guild.general, {}, { + api: { + async getStateEvent(roomID, type, key) { + called++ + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + t.equal(type, "m.room.power_levels") + t.equal(key, "") + return { + users: { + "@_ooye_bot:cadence.moe": 100 + } + } + }, + async getJoinedMembers(roomID) { + called++ + t.equal(roomID, "!kLRqKKUQXcibIMtOpl:cadence.moe") + return { + joined: { + "@_ooye_bot:cadence.moe": {display_name: null, avatar_url: null}, + "@user:matrix.org": {display_name: null, avatar_url: null} + } + } + } + } + }) t.deepEqual(events, [{ $type: "m.room.message", "m.mentions": {}, msgtype: "m.text", body: "#main", format: "org.matrix.custom.html", - formatted_body: '#main' + formatted_body: '#main' }]) + t.equal(called, 2) }) test("message2event: unknown room mention", async t => { From d673296619b753dd1780204c505f692349d81e78 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 12 Feb 2024 15:39:04 +1300 Subject: [PATCH 059/431] Embed text with pipe instead of arrow --- .../message-to-event.embeds.test.js | 36 +++++++++---------- d2m/converters/message-to-event.js | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/d2m/converters/message-to-event.embeds.test.js b/d2m/converters/message-to-event.embeds.test.js index 96ba53d..773f7b8 100644 --- a/d2m/converters/message-to-event.embeds.test.js +++ b/d2m/converters/message-to-event.embeds.test.js @@ -9,9 +9,9 @@ test("message2event embeds: nothing but a field", async t => { $type: "m.room.message", "m.mentions": {}, msgtype: "m.notice", - body: "> ### Amanda 🎵#2192 :online:" - + "\n> willow tree, branch 0" - + "\n> **❯ Uptime:**\n> 3m 55s\n> **❯ Memory:**\n> 64.45MB", + body: "| ### Amanda 🎵#2192 :online:" + + "\n| willow tree, branch 0" + + "\n| **❯ Uptime:**\n| 3m 55s\n| **❯ Memory:**\n| 64.45MB", format: "org.matrix.custom.html", formatted_body: '

Amanda 🎵#2192 \":online:\"' + '
willow tree, branch 0' @@ -26,14 +26,14 @@ test("message2event embeds: reply with just an embed", async t => { $type: "m.room.message", msgtype: "m.notice", "m.mentions": {}, - body: "> ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145" - + "\n> \n> ## https://twitter.com/i/status/1707484191963648161" - + "\n> \n> does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?" - + "\n> \n> ### Retweets" - + "\n> 119" - + "\n> \n> ### Likes" - + "\n> 5581" - + "\n> — Twitter", + body: "| ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145" + + "\n| \n| ## https://twitter.com/i/status/1707484191963648161" + + "\n| \n| does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?" + + "\n| \n| ### Retweets" + + "\n| 119" + + "\n| \n| ### Likes" + + "\n| 5581" + + "\n| — Twitter", format: "org.matrix.custom.html", formatted_body: '

⏺️ dynastic (@dynastic)

' + '

https://twitter.com/i/status/1707484191963648161' @@ -111,7 +111,7 @@ test("message2event embeds: blockquote in embed", async t => { }, { $type: "m.room.message", msgtype: "m.notice", - body: "> ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid\n> \n> reply draft\n> > The following is a message composed via consensus of the Stinker Council.\n> > \n> > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n> > \n> > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n> > \n> > There will be no further communication.\n> \n> [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid)", + body: "| ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid\n| \n| reply draft\n| > The following is a message composed via consensus of the Stinker Council.\n| > \n| > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n| > \n| > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n| > \n| > There will be no further communication.\n| \n| [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo?via=cadence.moe&via=example.invalid)", format: "org.matrix.custom.html", formatted_body: "

⏺️ minimus

reply draft

The following is a message composed via consensus of the Stinker Council.

For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.

Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.

There will be no further communication.

Go to Message

", "m.mentions": {} @@ -124,12 +124,12 @@ test("message2event embeds: crazy html is all escaped", async t => { t.deepEqual(events, [{ $type: "m.room.message", msgtype: "m.notice", - body: "> ## ⏺️ [Hey