From fff8f0d94c344c13a84463ced6e7c748d37f0141 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 14 Oct 2023 23:58:22 +1300 Subject: [PATCH 001/453] Review and upgrade dependencies --- package-lock.json | 162 ++++++++++++++++++++-------------------------- package.json | 8 +-- readme.md | 2 +- 3 files changed, 75 insertions(+), 97 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3847ee1..a005645 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,9 +10,9 @@ "license": "AGPL-3.0-or-later", "dependencies": { "@chriscdn/promise-semaphore": "^2.0.1", - "better-sqlite3": "^8.3.0", + "better-sqlite3": "^9.0.0", "chunk-text": "^2.0.1", - "cloudstorm": "^0.8.0", + "cloudstorm": "^0.9.0", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3", "entities": "^4.5.0", "giframe": "github:cloudrac3r/giframe#v0.4.1", @@ -25,7 +25,7 @@ "pngjs": "^7.0.0", "prettier-bytes": "^1.0.4", "sharp": "^0.32.6", - "snowtransfer": "^0.8.0", + "snowtransfer": "^0.9.0", "stream-mime-type": "^1.0.2", "try-to-catch": "^3.0.1", "turndown": "^7.1.2", @@ -36,7 +36,7 @@ "@types/node-fetch": "^2.6.3", "c8": "^8.0.1", "cross-env": "^7.0.3", - "discord-api-types": "^0.37.53", + "discord-api-types": "^0.37.60", "supertape": "^8.3.0", "tap-dot": "github:cloudrac3r/tap-dot#9dd7750ececeae3a96afba91905be812b6b2cc2d" } @@ -48,9 +48,9 @@ "dev": true }, "node_modules/@chriscdn/promise-semaphore": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@chriscdn/promise-semaphore/-/promise-semaphore-2.0.1.tgz", - "integrity": "sha512-C0Ku5DNZFbafbSRXagidIaRgzhgGmSHk4aAgPpmmHEostazBiSaMryovC/Aix3vRLNuaeGDKN/DHoNECmMD6jg==" + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@chriscdn/promise-semaphore/-/promise-semaphore-2.0.7.tgz", + "integrity": "sha512-xsa5SAYSBnYjqvGnzmaLca4X/RFeOl+ziCsIHl5iHkFBgE4NgWupB4z3A1rVMBM2I8TEKaah+5iu9Cm7gQu9JQ==" }, "node_modules/@cloudcmd/stub": { "version": "4.0.1", @@ -152,6 +152,14 @@ "node": ">=8" } }, + "node_modules/@fastify/busboy": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.0.0.tgz", + "integrity": "sha512-JUFJad5lv7jxj926GPgymrWQxxjPYuJNiNjNMzqT+HiuP6Vl3dk5xzG+8sTX96np0ZAluvaMzPsjhHZ5rNuNQQ==", + "engines": { + "node": ">=14" + } + }, "node_modules/@istanbuljs/schema": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", @@ -231,15 +239,15 @@ "dev": true }, "node_modules/@supertape/engine-loader": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@supertape/engine-loader/-/engine-loader-1.1.3.tgz", - "integrity": "sha512-5ilgEng0WBvMQjNJWQ/bnAA6HKgbLKxTya2C0RxFH0LYSN5faBVtgxjLDvTQ+5L+ZxjK/7ooQDDaRS1Mo0ga5Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@supertape/engine-loader/-/engine-loader-2.0.0.tgz", + "integrity": "sha512-1G2MmfZnSxx546omLPAVNgvG/iqOQZGiXHnjJ2JXKvuf2lpPdDRnNm5eLl81lvEG473zE9neX979TzeFcr3Dxw==", "dev": true, "dependencies": { "try-catch": "^3.0.0" }, "engines": { - "node": ">=14" + "node": ">=16" } }, "node_modules/@supertape/formatter-fail": { @@ -268,9 +276,9 @@ } }, "node_modules/@supertape/formatter-progress-bar": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@supertape/formatter-progress-bar/-/formatter-progress-bar-3.0.0.tgz", - "integrity": "sha512-rVFAQ21eApq3TQV8taFLNcCxcGZvvOPxQC63swdmHFCp+07Dt3tvC/aFxF35NLobc3rySasGSEuPucpyoPrjfg==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@supertape/formatter-progress-bar/-/formatter-progress-bar-4.1.0.tgz", + "integrity": "sha512-MYwso7kbiTE0DaZgbiSlNOikmEcFdL4RQUu1JvnW+cS6ZLl3fqNnmvKa1a14VChKyHzfaTKYLuqToN8zgUjP2g==", "dev": true, "dependencies": { "chalk": "^4.1.0", @@ -280,7 +288,7 @@ "once": "^1.4.0" }, "engines": { - "node": ">=14" + "node": ">=16" } }, "node_modules/@supertape/formatter-short": { @@ -325,19 +333,19 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.16.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.5.tgz", - "integrity": "sha512-seOA34WMo9KB+UA78qaJoCO20RJzZGVXQ5Sh6FWu0g/hfT44nKXnej3/tCQl7FL97idFpBhisLYCTB50S0EirA==", + "version": "18.18.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.5.tgz", + "integrity": "sha512-4slmbtwV59ZxitY4ixUZdy1uRLf9eSIvBWPQxNjhHYWEtn0FryfKpyS2cvADYXTayWdKEIsJengncrVvkI4I6A==", "dev": true }, "node_modules/@types/node-fetch": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.3.tgz", - "integrity": "sha512-ETTL1mOEdq/sxUtgtOhKjyB2Irra4cjxksvcMUR5Zr4n+PxVhsCD9WS46oPbHL3et9Zde7CNRr+WUNlcHvsX+w==", + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-95X8guJYhfqiuVVhRFxVQcf4hW/2bCuoPwDasMf/531STFoNoWTT7YDnWdXHEZKqAGUigmpG31r2FE70LwnzJw==", "dev": true, "dependencies": { "@types/node": "*", - "form-data": "^3.0.0" + "form-data": "^4.0.0" } }, "node_modules/@types/prop-types": { @@ -446,7 +454,8 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true }, "node_modules/available-typed-arrays": { "version": "1.0.5", @@ -512,13 +521,13 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "node_modules/better-sqlite3": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-8.4.0.tgz", - "integrity": "sha512-NmsNW1CQvqMszu/CFAJ3pLct6NEFlNfuGM6vw72KHkjOD1UDnL96XNN1BMQc1hiHo8vE2GbOWQYIpZ+YM5wrZw==", + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-9.0.0.tgz", + "integrity": "sha512-lDxQ9qg/XuUHZG6xzrQaMHkNWl37t35/LPB/VJGV8DdScSuGFNfFSqgscXEd8UIuyk/d9wU8iaMxQa4If5Wqog==", "hasInstallScript": true, "dependencies": { "bindings": "^1.5.0", - "prebuild-install": "^7.1.0" + "prebuild-install": "^7.1.1" } }, "node_modules/bindings": { @@ -631,17 +640,6 @@ "ieee754": "^1.2.1" } }, - "node_modules/busboy": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", - "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", - "dependencies": { - "streamsearch": "^1.1.0" - }, - "engines": { - "node": ">=10.16.0" - } - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -784,14 +782,14 @@ } }, "node_modules/cloudstorm": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.8.3.tgz", - "integrity": "sha512-4c2rqFFvzM4P3pcnjnGUlYuyBjx/xnMew6imB0sFwmNLITLCTLYa3qGkrnhI1g/tM0fqg+Gr+EmDHiDZfEr9LQ==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/cloudstorm/-/cloudstorm-0.9.0.tgz", + "integrity": "sha512-n5M5TVnvm/X5vdNKy85q8muMregnvPWxv7HGSDCChL/FReOh2PGOm0FZJVm4hcB+KIM07KmiJTiCSQTnrTrSnQ==", "dependencies": { - "snowtransfer": "^0.8.3" + "snowtransfer": "^0.9.0" }, "engines": { - "node": ">=12.0.0" + "node": ">=14.8.0" } }, "node_modules/color": { @@ -841,6 +839,7 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -1014,6 +1013,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, "engines": { "node": ">=0.4.0" } @@ -1053,9 +1053,9 @@ } }, "node_modules/discord-api-types": { - "version": "0.37.53", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.53.tgz", - "integrity": "sha512-N6uUgv50OyP981Mfxrrt0uxcqiaNr0BDaQIoqfk+3zM2JpZtwU9v7ce1uaFAP53b2xSDvcbrk80Kneui6XJgGg==" + "version": "0.37.60", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.60.tgz", + "integrity": "sha512-5BELXTsv7becqVHrD81nZrqT4oEyXXWBwbsO/kwDDu6X3u19VV1tYDB5I5vaVAK+c1chcDeheI9zACBLm41LiQ==" }, "node_modules/discord-markdown": { "version": "2.4.1", @@ -1345,9 +1345,9 @@ } }, "node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", "dev": true, "dependencies": { "asynckit": "^0.4.0", @@ -1549,9 +1549,9 @@ } }, "node_modules/heatsync": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/heatsync/-/heatsync-2.4.1.tgz", - "integrity": "sha512-cRzLwnKnJ5O4dQWXiJyFp4myKY8lGfK+49/SbPsvnr3pf2PNG1Xh8pPono303cjJeFpaPSTs609mQH1xhPVyzA==", + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/heatsync/-/heatsync-2.4.2.tgz", + "integrity": "sha512-s+YzwGpCjsJLRCuz6Ur8JFKz7vXEMFAPLqDbaEvMR5Um/IPPJpmupBH7LKeiyfGkIScFz9iyBPa1TifcoU4D7A==", "dependencies": { "backtracker": "3.3.2" } @@ -2192,9 +2192,9 @@ "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" }, "node_modules/node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "dependencies": { "whatwg-url": "^5.0.0" }, @@ -2922,29 +2922,15 @@ } }, "node_modules/snowtransfer": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.8.3.tgz", - "integrity": "sha512-0X6NLFBUKppYT5VH/mVQNGX+ufv0AndunZC84MqGAR/3rfTIGQblgGJlHlDQbeCytlXdMpgRHIGQnBFlE094NQ==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/snowtransfer/-/snowtransfer-0.9.0.tgz", + "integrity": "sha512-43Q0pvk7ZV8uZwcL/IhEFYKFZj53FOqxr2dVDwduPT87eHOJzfs8aQ+tNDqsjW6OMUBurwR3XZZFEpQ2f/XzXA==", "dependencies": { - "discord-api-types": "^0.37.47", - "form-data": "^4.0.0", - "undici": "^5.22.1" + "discord-api-types": "^0.37.60", + "undici": "^5.26.3" }, "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/snowtransfer/node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" + "node": ">=14.18.0" } }, "node_modules/source-map": { @@ -3022,14 +3008,6 @@ "node": ">=10" } }, - "node_modules/streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/streamx": { "version": "2.15.1", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz", @@ -3122,18 +3100,18 @@ } }, "node_modules/supertape": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/supertape/-/supertape-8.3.0.tgz", - "integrity": "sha512-dcMylmkr1Mctr5UBCrlvZynuBRuLvlkWJLGXdL/PcI41BERnObO+kV0PeZhH5n6lwVnvK2xfvZyN32WIAPf/tw==", + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/supertape/-/supertape-8.6.0.tgz", + "integrity": "sha512-zvAXZaliVu8qpGRx5KiYQfPZcQD9B361lmRtXb3zyilpHHc0/5ygQ9MfWYZEwXywxHDfve3w8ZukI/NKPT9PyA==", "dev": true, "dependencies": { "@cloudcmd/stub": "^4.0.0", "@putout/cli-keypress": "^1.0.0", "@putout/cli-validate-args": "^1.0.1", - "@supertape/engine-loader": "^1.0.0", + "@supertape/engine-loader": "^2.0.0", "@supertape/formatter-fail": "^3.0.0", "@supertape/formatter-json-lines": "^2.0.0", - "@supertape/formatter-progress-bar": "^3.0.0", + "@supertape/formatter-progress-bar": "^4.0.0", "@supertape/formatter-short": "^2.0.0", "@supertape/formatter-tap": "^3.0.0", "@supertape/operator-stub": "^3.0.0", @@ -3414,11 +3392,11 @@ } }, "node_modules/undici": { - "version": "5.22.1", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.22.1.tgz", - "integrity": "sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw==", + "version": "5.26.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.3.tgz", + "integrity": "sha512-H7n2zmKEWgOllKkIUkLvFmsJQj062lSm3uA4EYApG8gLuiOM0/go9bIoC3HVaSnfg4xunowDE2i9p8drkXuvDw==", "dependencies": { - "busboy": "^1.6.0" + "@fastify/busboy": "^2.0.0" }, "engines": { "node": ">=14.0" diff --git a/package.json b/package.json index 6a9deea..29f6f1c 100644 --- a/package.json +++ b/package.json @@ -16,9 +16,9 @@ "license": "AGPL-3.0-or-later", "dependencies": { "@chriscdn/promise-semaphore": "^2.0.1", - "better-sqlite3": "^8.3.0", + "better-sqlite3": "^9.0.0", "chunk-text": "^2.0.1", - "cloudstorm": "^0.8.0", + "cloudstorm": "^0.9.0", "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3", "entities": "^4.5.0", "giframe": "github:cloudrac3r/giframe#v0.4.1", @@ -31,7 +31,7 @@ "pngjs": "^7.0.0", "prettier-bytes": "^1.0.4", "sharp": "^0.32.6", - "snowtransfer": "^0.8.0", + "snowtransfer": "^0.9.0", "stream-mime-type": "^1.0.2", "try-to-catch": "^3.0.1", "turndown": "^7.1.2", @@ -42,7 +42,7 @@ "@types/node-fetch": "^2.6.3", "c8": "^8.0.1", "cross-env": "^7.0.3", - "discord-api-types": "^0.37.53", + "discord-api-types": "^0.37.60", "supertape": "^8.3.0", "tap-dot": "github:cloudrac3r/tap-dot#9dd7750ececeae3a96afba91905be812b6b2cc2d" }, diff --git a/readme.md b/readme.md index 5c97745..61d1951 100644 --- a/readme.md +++ b/readme.md @@ -73,7 +73,7 @@ You'll need: Follow these steps: -1. [Get Node.js version 18 or later](https://nodejs.org/en/download/releases) (the version is required by the matrix-appservice dependency) +1. [Get Node.js version 18 or later](https://nodejs.org/en/download/releases) (the version is required by the better-sqlite3 and matrix-appservice dependencies) 1. Clone this repo and checkout a specific tag. (Development happens on main. Stabler versions are tagged.) From 9c3f1abd3a2c12d30b57bdec55d85ad352a5dab5 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sun, 15 Oct 2023 00:26:52 +1300 Subject: [PATCH 002/453] Upload files to Discord as streams for speed --- m2d/actions/channel-webhook.js | 5 ++-- m2d/actions/send-event.js | 40 +++++++++++++++--------------- m2d/converters/event-to-message.js | 5 ++-- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/m2d/actions/channel-webhook.js b/m2d/actions/channel-webhook.js index 3bb728d..52b4095 100644 --- a/m2d/actions/channel-webhook.js +++ b/m2d/actions/channel-webhook.js @@ -2,6 +2,7 @@ const assert = require("assert").strict const DiscordTypes = require("discord-api-types/v10") +const {Readable} = require("stream") const passthrough = require("../../passthrough") const {discord, db, select} = passthrough @@ -51,7 +52,7 @@ async function withWebhook(channelID, callback) { /** * @param {string} channelID - * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[]}} data + * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]}} data * @param {string} [threadID] */ async function sendMessageWithWebhook(channelID, data, threadID) { @@ -64,7 +65,7 @@ async function sendMessageWithWebhook(channelID, data, threadID) { /** * @param {string} channelID * @param {string} messageID - * @param {DiscordTypes.RESTPatchAPIWebhookWithTokenMessageJSONBody & {files?: {name: string, file: Buffer}[]}} data + * @param {DiscordTypes.RESTPatchAPIWebhookWithTokenMessageJSONBody & {files?: {name: string, file: Buffer | Readable}[]}} data * @param {string} [threadID] */ async function editMessageWithWebhook(channelID, messageID, data, threadID) { diff --git a/m2d/actions/send-event.js b/m2d/actions/send-event.js index 13b5650..9b3fbec 100644 --- a/m2d/actions/send-event.js +++ b/m2d/actions/send-event.js @@ -1,11 +1,11 @@ // @ts-check -const assert = require("assert").strict -const crypto = require("crypto") -const {pipeline} = require("stream") -const {promisify} = require("util") const Ty = require("../../types") const DiscordTypes = require("discord-api-types/v10") +const {Readable} = require("stream") +const assert = require("assert").strict +const crypto = require("crypto") +const fetch = require("node-fetch").default const passthrough = require("../../passthrough") const {sync, discord, db, select} = passthrough @@ -17,13 +17,12 @@ const eventToMessage = sync.require("../converters/event-to-message") const api = sync.require("../../matrix/api") /** - * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer})[]}} message - * @returns {Promise} + * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer | Readable})[]}} message + * @returns {Promise} */ async function resolvePendingFiles(message) { if (!message.pendingFiles) return message const files = await Promise.all(message.pendingFiles.map(async p => { - let fileBuffer if ("buffer" in p) { return { name: p.name, @@ -31,21 +30,22 @@ async function resolvePendingFiles(message) { } } if ("key" in p) { - // Encrypted + // Encrypted file const d = crypto.createDecipheriv("aes-256-ctr", Buffer.from(p.key, "base64url"), Buffer.from(p.iv, "base64url")) - fileBuffer = await fetch(p.url).then(res => res.arrayBuffer()).then(x => { - return Buffer.concat([ - d.update(Buffer.from(x)), - d.final() - ]) - }) + // @ts-ignore + fetch(p.url).then(res => res.body.pipe(d)) + return { + name: p.name, + file: d + } } else { - // Unencrypted - fileBuffer = await fetch(p.url).then(res => res.arrayBuffer()).then(x => Buffer.from(x)) - } - return { - name: p.name, - file: fileBuffer // TODO: Once SnowTransfer supports ReadableStreams for attachment uploads, pass in those instead of Buffers + // Unencrypted file + /** @type {Readable} */ // @ts-ignore + const body = await fetch(p.url).then(res => res.body) + return { + name: p.name, + file: body + } } })) const newMessage = { diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index 5f6f3e6..8907508 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -2,6 +2,7 @@ const Ty = require("../../types") const DiscordTypes = require("discord-api-types/v10") +const {Readable} = require("stream") const chunk = require("chunk-text") const TurndownService = require("turndown") const assert = require("assert").strict @@ -9,8 +10,6 @@ const entities = require("entities") const passthrough = require("../../passthrough") const {sync, db, discord, select, from} = passthrough -/** @type {import("../../matrix/file")} */ -const file = sync.require("../../matrix/file") /** @type {import("../converters/utils")} */ const utils = sync.require("../converters/utils") /** @type {import("./emoji-sheet")} */ @@ -245,7 +244,7 @@ async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles) * @param {{api: import("../../matrix/api")}} di simple-as-nails dependency injection for the matrix API */ async function eventToMessage(event, guild, di) { - /** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[]})[]} */ + /** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */ let messages = [] let displayName = event.sender From a542bbdca79ba0825e08fe589f85224e212c9b88 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 16 Oct 2023 16:24:48 +1300 Subject: [PATCH 003/453] Update discord-markdown to support list and header --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index a005645..11585f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "better-sqlite3": "^9.0.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.9.0", - "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3", + "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", "giframe": "github:cloudrac3r/giframe#v0.4.1", "heatsync": "^2.4.1", @@ -1058,8 +1058,8 @@ "integrity": "sha512-5BELXTsv7becqVHrD81nZrqT4oEyXXWBwbsO/kwDDu6X3u19VV1tYDB5I5vaVAK+c1chcDeheI9zACBLm41LiQ==" }, "node_modules/discord-markdown": { - "version": "2.4.1", - "resolved": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3", + "version": "2.5.1", + "resolved": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "license": "MIT", "dependencies": { "simple-markdown": "^0.7.2" diff --git a/package.json b/package.json index 29f6f1c..42988c8 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "better-sqlite3": "^9.0.0", "chunk-text": "^2.0.1", "cloudstorm": "^0.9.0", - "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#abc56d544072a1dc5624adfea455b0e902adf7b3", + "discord-markdown": "git+https://git.sr.ht/~cadence/nodejs-discord-markdown#de519353668c87ecc8c543e9749093481bc72ff8", "entities": "^4.5.0", "giframe": "github:cloudrac3r/giframe#v0.4.1", "heatsync": "^2.4.1", From 762e48230c682fb9256119a51d600dc6bbafbadf Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 16 Oct 2023 16:47:42 +1300 Subject: [PATCH 004/453] Ensure the appservice bot user is registered Synapse does it automatically, but Conduit requires this HTTP call. --- d2m/actions/register-user.js | 2 +- scripts/seed.js | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/d2m/actions/register-user.js b/d2m/actions/register-user.js index 9b5527f..b605c3a 100644 --- a/d2m/actions/register-user.js +++ b/d2m/actions/register-user.js @@ -36,7 +36,7 @@ async function createSim(user) { await api.register(localpart) } catch (e) { // If user creation fails, manually undo the database change. Still isn't perfect, but should help. - // (A transaction would be preferable, but I don't think it's safe to leave transaction open across event loop ticks.) + // (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.) db.prepare("DELETE FROM sim WHERE user_id = ?").run(user.id) throw e } diff --git a/scripts/seed.js b/scripts/seed.js index 0f2f23d..4fc704c 100644 --- a/scripts/seed.js +++ b/scripts/seed.js @@ -67,6 +67,18 @@ async function uploadAutoEmoji(guild, name, filename) { console.log("✅ Database is ready...") + // ensure appservice bot user is registered... + try { + await api.register(reg.sender_localpart) + } catch (e) { + if (e.data?.error !== "Internal server error") throw e // "Internal server error" is the only OK error because Synapse says this if you try to register the same username twice. + } + + // upload initial images... + const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png") + + console.log("✅ Matrix appservice login works...") + // upload the L1 L2 emojis to some guild const emojis = db.prepare("SELECT name FROM auto_emoji WHERE name = 'L1' OR name = 'L2'").pluck().all() if (emojis.length !== 2) { @@ -104,11 +116,6 @@ async function uploadAutoEmoji(guild, name, filename) { } console.log("✅ Emojis are ready...") - // ensure homeserver well-known is valid and returns reg.ooye.server_name... - - // upload initial images... - const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png") - // set profile data on discord... const avatarImageBuffer = await fetch("https://cadence.moe/friends/out_of_your_element.png").then(res => res.arrayBuffer()) await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + Buffer.from(avatarImageBuffer).toString("base64")}) From afbbe0da3d9bf561e3f5394c0fb6009a5d9f6711 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 28 Oct 2023 00:24:42 +1300 Subject: [PATCH 005/453] Fix more edge-case embed formatting --- .../message-to-event.embeds.test.js | 68 ++++++-- d2m/converters/message-to-event.js | 149 ++++++++++++------ m2d/converters/emoji.js | 1 + m2d/converters/event-to-message.js | 18 +++ m2d/converters/event-to-message.test.js | 43 ++++- m2d/converters/utils.js | 65 ++++++++ matrix/matrix-command-handler.js | 53 +------ package-lock.json | 23 ++- package.json | 4 +- readme.md | 8 +- test/data.js | 108 +++++++++++++ test/ooye-test-data.sql | 15 +- 12 files changed, 428 insertions(+), 127 deletions(-) diff --git a/d2m/converters/message-to-event.embeds.test.js b/d2m/converters/message-to-event.embeds.test.js index 93d189c..173e016 100644 --- a/d2m/converters/message-to-event.embeds.test.js +++ b/d2m/converters/message-to-event.embeds.test.js @@ -35,14 +35,14 @@ test("message2event embeds: nothing but a field", async t => { $type: "m.room.message", "m.mentions": {}, msgtype: "m.notice", - body: "> **Amanda 🎵#2192 :online:" - + "\n> willow tree, branch 0**" + body: "> ### Amanda 🎵#2192 :online:" + + "\n> willow tree, branch 0" + "\n> **❯ Uptime:**\n> 3m 55s\n> **❯ Memory:**\n> 64.45MB", format: "org.matrix.custom.html", - formatted_body: '
Amanda 🎵#2192 \":online:\"' + formatted_body: '

Amanda 🎵#2192 \":online:\"' + '
willow tree, branch 0' + '
❯ Uptime:
3m 55s' - + '
❯ Memory:
64.45MB

' + + '
❯ Memory:
64.45MB

' }]) }) @@ -52,19 +52,19 @@ test("message2event embeds: reply with just an embed", async t => { $type: "m.room.message", msgtype: "m.notice", "m.mentions": {}, - body: "> [**⏺️ dynastic (@dynastic)**](https://twitter.com/i/user/719631291747078145)" - + "\n> \n> **https://twitter.com/i/status/1707484191963648161**" + body: "> ## ⏺️ dynastic (@dynastic) https://twitter.com/i/user/719631291747078145" + + "\n> \n> ## https://twitter.com/i/status/1707484191963648161" + "\n> \n> does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?" - + "\n> \n> **Retweets**" + + "\n> \n> ### Retweets" + "\n> 119" - + "\n> \n> **Likes**" + + "\n> \n> ### Likes" + "\n> 5581" - + "\n> \n> — Twitter", + + "\n> — Twitter", format: "org.matrix.custom.html", - formatted_body: '
⏺️ dynastic (@dynastic)' - + '

https://twitter.com/i/status/1707484191963648161' - + '

does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?' - + '

Retweets
119

Likes
5581

— Twitter
' + formatted_body: '

⏺️ dynastic (@dynastic)

' + + '

https://twitter.com/i/status/1707484191963648161' + + '

does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?' + + '

Retweets
119

Likes
5581

— Twitter
' }]) }) @@ -99,3 +99,45 @@ test("message2event embeds: image embed and attachment", async t => { "m.mentions": {} }]) }) + +test("message2event embeds: blockquote in embed", async t => { + const events = await messageToEvent(data.message_with_embeds.blockquote_in_embed, data.guild.general) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.text", + body: ":emoji: **4 |** #wonderland", + format: "org.matrix.custom.html", + formatted_body: `\":emoji:\" 4 | #wonderland`, + "m.mentions": {} + }, { + $type: "m.room.message", + msgtype: "m.notice", + body: "> ## ⏺️ minimus https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo\n> \n> reply draft\n> > The following is a message composed via consensus of the Stinker Council.\n> > \n> > For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.\n> > \n> > Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.\n> > \n> > There will be no further communication.\n> \n> [Go to Message](https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$dVCLyj6kxb3DaAWDtjcv2kdSny8JMMHdDhCMz8mDxVo)", + format: "org.matrix.custom.html", + formatted_body: "

⏺️ minimus

reply draft

The following is a message composed via consensus of the Stinker Council.

For those who are not currently aware of our existence, we represent the organization known as Wonderland. Our previous mission centered around the assortment and study of puzzling objects, entities and other assorted phenomena. This mission was the focus of our organization for more than 28 years.

Due to circumstances outside of our control, this directive has now changed. Our new mission will be the extermination of the stinker race.

There will be no further communication.

Go to Message

", + "m.mentions": {} + }]) +}) + +test("message2event embeds: crazy html is all escaped", async t => { + const events = await messageToEvent(data.message_with_embeds.escaping_crazy_html_tags, data.guild.general) + t.deepEqual(events, [{ + $type: "m.room.message", + msgtype: "m.notice", + body: "> ## ⏺️ [Hey