sender_workers scaling. this time, with feeling!

scale rocksdb background jobs and subcompactions

change rocksdb default error level to info from error

delete unused num_threads function

fix warns from cargo

.forgejo/workflows/release-image.yml

@@ -49,6 +49,7 @@ jobs:
             const platforms = ['linux/amd64', 'linux/arm64']
             core.setOutput('build_matrix', JSON.stringify({
               platform: platforms,
+              target_cpu: ['base'],
               include: platforms.map(platform => { return {
                 platform,
                 slug: platform.replace('/', '-')
@@ -66,6 +67,8 @@ jobs:
     strategy:
       matrix:
         {
+          "target_cpu": ["base"],
+          "profile": ["release"],
           "include":
             [
               { "platform": "linux/amd64", "slug": "linux-amd64" },
@@ -73,6 +76,7 @@ jobs:
             ],
           "platform": ["linux/amd64", "linux/arm64"],
         }
+
     steps:
       - name: Echo strategy
         run: echo '${{ toJSON(fromJSON(needs.define-variables.outputs.build_matrix)) }}'
@@ -140,8 +144,8 @@ jobs:
         uses: actions/cache@v3
         with:
           path: |
-            cargo-target-${{ matrix.slug }}
-          key: cargo-target-${{ matrix.slug }}-${{hashFiles('**/Cargo.lock') }}-${{steps.rust-toolchain.outputs.rustc_version}}
+            cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}
+          key: cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}-${{hashFiles('**/Cargo.lock') }}-${{steps.rust-toolchain.outputs.rustc_version}}
       - name: Cache apt cache
         id: cache-apt
         uses: actions/cache@v3
@@ -163,9 +167,9 @@ jobs:
             {
               ".cargo/registry": "/usr/local/cargo/registry",
               ".cargo/git/db": "/usr/local/cargo/git/db",
-              "cargo-target-${{ matrix.slug }}": {
+              "cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}": {
                 "target": "/app/target",
-                "id": "cargo-target-${{ matrix.platform }}"
+                "id": "cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}"
               },
               "var-cache-apt-${{ matrix.slug }}": "/var/cache/apt",
               "var-lib-apt-${{ matrix.slug }}": "/var/lib/apt"

docker/Dockerfile

@@ -1,15 +1,16 @@
 ARG RUST_VERSION=1
+ARG DEBIAN_VERSION=bookworm
 
 FROM --platform=$BUILDPLATFORM docker.io/tonistiigi/xx AS xx
-FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-bookworm AS base
-FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-bookworm AS toolchain
+FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-${DEBIAN_VERSION} AS base
+FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-${DEBIAN_VERSION} AS toolchain
 
 # Prevent deletion of apt cache
 RUN rm -f /etc/apt/apt.conf.d/docker-clean
 
 # Match Rustc version as close as possible
 # rustc -vV
-ARG LLVM_VERSION=19
+ARG LLVM_VERSION=20
 # ENV RUSTUP_TOOLCHAIN=${RUST_VERSION}
 
 # Install repo tools
@@ -19,10 +20,18 @@ ARG LLVM_VERSION=19
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
     apt-get update && apt-get install -y \
-    clang-${LLVM_VERSION} lld-${LLVM_VERSION} pkg-config make jq \
-    curl git \
+    pkg-config make jq \
+    curl git software-properties-common \
     file
 
+# LLVM packages
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    curl https://apt.llvm.org/llvm.sh > llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh ${LLVM_VERSION} && \
+    rm llvm.sh
+
 # Create symlinks for LLVM tools
 RUN <<EOF
     set -o xtrace
@@ -39,7 +48,7 @@ EOF
 
 # Developer tool versions
 # renovate: datasource=github-releases depName=cargo-bins/cargo-binstall
-ENV BINSTALL_VERSION=1.12.3
+ENV BINSTALL_VERSION=1.13.0
 # renovate: datasource=github-releases depName=psastras/sbom-rs
 ENV CARGO_SBOM_VERSION=0.9.1
 # renovate: datasource=crate depName=lddtree
@@ -140,11 +149,12 @@ ENV GIT_REMOTE_COMMIT_URL=$GIT_REMOTE_COMMIT_URL
 ENV CONDUWUIT_VERSION_EXTRA=$CONDUWUIT_VERSION_EXTRA
 ENV CONTINUWUITY_VERSION_EXTRA=$CONTINUWUITY_VERSION_EXTRA
 
+ARG RUST_PROFILE=release
 
 # Build the binary
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git/db \
-    --mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM} \
+    --mount=type=cache,target=/app/target,id=cargo-target-${TARGET_CPU}-${TARGETPLATFORM}-${RUST_PROFILE} \
     bash <<'EOF'
     set -o allexport
     set -o xtrace
@@ -153,7 +163,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
             jq -r ".target_directory"))
     mkdir /out/sbin
     PACKAGE=conduwuit
-    xx-cargo build --locked --release \
+    xx-cargo build --locked --profile ${RUST_PROFILE} \
         -p $PACKAGE;
     BINARIES=($(cargo metadata --no-deps --format-version 1 | \
         jq -r ".packages[] | select(.name == \"$PACKAGE\") | .targets[] | select( .kind | map(. == \"bin\") | any ) | .name"))

docs/deploying/arch-linux.md

@@ -1,3 +1,5 @@
 # Continuwuity for Arch Linux
 
-Continuwuity does not have any Arch Linux packages at this time.
+Continuwuity is available on the `archlinuxcn` repository and AUR, with the same package name `continuwuity`, which includes latest taggged version. The development version is available on AUR as `continuwuity-git`
+
+Simply install the `continuwuity` package. Configure the service in `/etc/conduwuit/conduwuit.toml`, then enable/start the continuwuity.service.

rust-toolchain.toml

@@ -9,7 +9,7 @@
 # If you're having trouble making the relevant changes, bug a maintainer.
 
 [toolchain]
-channel = "1.86.0"
+channel = "1.87.0"
 profile = "minimal"
 components = [
     # For rust-analyzer
@@ -19,11 +19,3 @@ components = [
     "rustfmt",
     "clippy",
 ]
-targets = [
-    #"x86_64-apple-darwin",
-    "x86_64-unknown-linux-gnu",
-    "x86_64-unknown-linux-musl",
-    "aarch64-unknown-linux-musl",
-    "aarch64-unknown-linux-gnu",
-    #"aarch64-apple-darwin",
-]

src/admin/debug/commands.rs

@@ -239,10 +239,11 @@ pub(super) async fn get_remote_pdu(
 		})
 		.await
 	{
-		| Err(e) =>
+		| Err(e) => {
 			return Err!(
 				"Remote server did not have PDU or failed sending request to remote server: {e}"
-			),
+			);
+		},
 		| Ok(response) => {
 			let json: CanonicalJsonObject =
 				serde_json::from_str(response.pdu.get()).map_err(|e| {
@@ -384,8 +385,9 @@ pub(super) async fn change_log_level(&self, filter: Option<String>, reset: bool)
 			.reload
 			.reload(&old_filter_layer, Some(handles))
 		{
-			| Err(e) =>
-				return Err!("Failed to modify and reload the global tracing log level: {e}"),
+			| Err(e) => {
+				return Err!("Failed to modify and reload the global tracing log level: {e}");
+			},
 			| Ok(()) => {
 				let value = &self.services.server.config.log;
 				let out = format!("Successfully changed log level back to config value {value}");
@@ -408,8 +410,9 @@ pub(super) async fn change_log_level(&self, filter: Option<String>, reset: bool)
 			.reload(&new_filter_layer, Some(handles))
 		{
 			| Ok(()) => return self.write_str("Successfully changed log level").await,
-			| Err(e) =>
-				return Err!("Failed to modify and reload the global tracing log level: {e}"),
+			| Err(e) => {
+				return Err!("Failed to modify and reload the global tracing log level: {e}");
+			},
 		}
 	}
 
@@ -529,6 +532,7 @@ pub(super) async fn force_set_room_state_from_server(
 	&self,
 	room_id: OwnedRoomId,
 	server_name: OwnedServerName,
+	at_event: Option<OwnedEventId>,
 ) -> Result {
 	if !self
 		.services
@@ -540,13 +544,18 @@ pub(super) async fn force_set_room_state_from_server(
 		return Err!("We are not participating in the room / we don't know about the room ID.");
 	}
 
-	let first_pdu = self
-		.services
-		.rooms
-		.timeline
-		.latest_pdu_in_room(&room_id)
-		.await
-		.map_err(|_| err!(Database("Failed to find the latest PDU in database")))?;
+	let at_event_id = match at_event {
+		| Some(event_id) => event_id,
+		| None => self
+			.services
+			.rooms
+			.timeline
+			.latest_pdu_in_room(&room_id)
+			.await
+			.map_err(|_| err!(Database("Failed to find the latest PDU in database")))?
+			.event_id
+			.clone(),
+	};
 
 	let room_version = self.services.rooms.state.get_room_version(&room_id).await?;
 
@@ -557,7 +566,7 @@ pub(super) async fn force_set_room_state_from_server(
 		.sending
 		.send_federation_request(&server_name, get_room_state::v1::Request {
 			room_id: room_id.clone(),
-			event_id: first_pdu.event_id.clone(),
+			event_id: at_event_id,
 		})
 		.await?;
 

src/admin/debug/mod.rs

@@ -177,6 +177,9 @@ pub(super) enum DebugCommand {
 		room_id: OwnedRoomId,
 		/// The server we will use to query the room state for
 		server_name: OwnedServerName,
+		/// The event ID of the latest known PDU in the room. Will be found
+		/// automatically if not provided.
+		event_id: Option<OwnedEventId>,
 	},
 
 	/// - Runs a server name through conduwuit's true destination resolution

src/core/config/mod.rs

@@ -1823,9 +1823,9 @@ pub struct Config {
 	pub stream_amplification: usize,
 
 	/// Number of sender task workers; determines sender parallelism. Default is
-	/// '4'. Override by setting a different value. Values clamped 1 to core count.
+	/// core count. Override by setting a different value.
 	///
-	/// default: 4
+	/// default: core count
 	#[serde(default = "default_sender_workers")]
 	pub sender_workers: usize,
 
@@ -2312,7 +2312,7 @@ fn default_stream_width_scale() -> f32 { 1.0 }
 
 fn default_stream_amplification() -> usize { 1024 }
 
-fn default_sender_workers() -> usize { 4 }
+fn default_sender_workers() -> usize { parallelism_scaled(1) }
 
 fn default_client_receive_timeout() -> u64 { 75 }
 

src/service/rooms/timeline/mod.rs

@@ -976,9 +976,8 @@ impl Service {
 		state_lock: &'a RoomMutexGuard,
 	) -> Result<Option<RawPduId>>
 	where
-		Leaves: Iterator<Item = &'a EventId> + Send + Clone + 'a,
+		Leaves: Iterator<Item = &'a EventId> + Send + 'a,
 	{
-		assert!(new_room_leaves.clone().count() > 0, "extremities are empty");
 		// We append to state before appending the pdu, so we don't have a moment in
 		// time with the pdu without it's state. This is okay because append_pdu can't
 		// fail.