From 1e9701f379053b6f20681b48db6ede5026db4681 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 7 Sep 2025 18:59:05 +0100 Subject: [PATCH 1/6] ci(release-image): Skip setup steps when using persistent BuildKit When BUILDKIT_ENDPOINT is set, builds run on a persistent BuildKit instance, making runner setup steps unnecessary. Skip Rust toolchain installation, QEMU setup, caching steps, and timelord to eliminate ~7 operations per job. Also adds output to git SHA and timestamp steps for visibility. Cuts at least a minute off average build time through fewer installs, cache restores, and cache saves. --- .forgejo/workflows/release-image.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 52f5e6e0..1a0e4f4e 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -101,6 +101,7 @@ jobs: with: persist-credentials: false - name: Install rust + if: ${{ env.BUILDKIT_ENDPOINT == '' }} id: rust-toolchain uses: ./.forgejo/actions/rust-toolchain @@ -111,6 +112,7 @@ jobs: driver: ${{ env.BUILDKIT_ENDPOINT != '' && 'remote' || 'docker-container' }} endpoint: ${{ env.BUILDKIT_ENDPOINT || '' }} - name: Set up QEMU + if: ${{ env.BUILDKIT_ENDPOINT == '' }} uses: docker/setup-qemu-action@v3 # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Login to builtin registry @@ -140,15 +142,21 @@ jobs: run: | calculatedSha=$(git rev-parse --short ${{ github.sha }}) echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV + echo "Short SHA: $calculatedSha" - name: Get Git commit timestamps - run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV + run: | + timestamp=$(git log -1 --pretty=%ct) + echo "TIMESTAMP=$timestamp" >> $GITHUB_ENV + echo "Commit timestamp: $timestamp" - uses: ./.forgejo/actions/timelord + if: ${{ env.BUILDKIT_ENDPOINT == '' }} with: key: timelord-v0 path: . - name: Cache Rust registry + if: ${{ env.BUILDKIT_ENDPOINT == '' }} uses: actions/cache@v3 with: path: | @@ -158,6 +166,7 @@ jobs: .cargo/registry/src key: rust-registry-image-${{hashFiles('**/Cargo.lock') }} - name: Cache cargo target + if: ${{ env.BUILDKIT_ENDPOINT == '' }} id: cache-cargo-target uses: actions/cache@v3 with: @@ -165,6 +174,7 @@ jobs: cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }} key: cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}-${{hashFiles('**/Cargo.lock') }}-${{steps.rust-toolchain.outputs.rustc_version}} - name: Cache apt cache + if: ${{ env.BUILDKIT_ENDPOINT == '' }} id: cache-apt uses: actions/cache@v3 with: @@ -172,6 +182,7 @@ jobs: var-cache-apt-${{ matrix.slug }} key: var-cache-apt-${{ matrix.slug }} - name: Cache apt lib + if: ${{ env.BUILDKIT_ENDPOINT == '' }} id: cache-apt-lib uses: actions/cache@v3 with: @@ -179,6 +190,7 @@ jobs: var-lib-apt-${{ matrix.slug }} key: var-lib-apt-${{ matrix.slug }} - name: inject cache into docker + if: ${{ env.BUILDKIT_ENDPOINT == '' }} uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.3.0 with: cache-map: | From 4aeb9b721f26178062f7d9d898a6517f44d819a8 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 3 Sep 2025 14:20:50 +0100 Subject: [PATCH 2/6] feat(PR977): Support omitting members in the send_join response --- src/api/server/send_join.rs | 97 ++++++++++++++++++----- src/service/rooms/state_compressor/mod.rs | 2 +- 2 files changed, 80 insertions(+), 19 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 652451c7..3aebbbe7 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -1,12 +1,13 @@ #![allow(deprecated)] -use std::borrow::Borrow; +use std::{borrow::Borrow, time::Instant}; use axum::extract::State; use conduwuit::{ - Err, Result, at, err, + Err, Event, Result, at, debug, err, info, matrix::event::gen_event_id_canonical_json, - utils::stream::{IterStream, TryBroadbandExt}, + trace, + utils::stream::{BroadbandExt, IterStream, TryBroadbandExt}, warn, }; use conduwuit_service::Services; @@ -25,12 +26,14 @@ use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; use crate::Ruma; /// helper method for /send_join v1 and v2 +#[tracing::instrument(skip(services, pdu, omit_members), fields(room_id = room_id.as_str(), origin = origin.as_str()))] async fn create_join_event( services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, -) -> Result { + omit_members: bool, +) -> Result { if !services.rooms.metadata.exists(room_id).await { return Err!(Request(NotFound("Room is unknown to this server."))); } @@ -201,6 +204,7 @@ async fn create_join_event( .lock(room_id) .await; + debug!("Acquired send_join mutex, persisting join event"); let pdu_id = services .rooms .event_handler @@ -210,7 +214,7 @@ async fn create_join_event( .ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?; drop(mutex_lock); - + debug!("Fetching current state IDs"); let state_ids: Vec = services .rooms .state_accessor @@ -219,9 +223,25 @@ async fn create_join_event( .collect() .await; + #[allow(clippy::unnecessary_unwrap)] let state = state_ids .iter() .try_stream() + .broad_filter_map(|event_id| async move { + if omit_members && event_id.is_ok() { + let pdu = services + .rooms + .timeline + .get_pdu(event_id.as_ref().unwrap()) + .await; + if pdu.is_ok_and(|p| p.kind().to_cow_str() == "m.room.member") { + trace!("omitting member event {event_id:?} from returned state"); + // skip members + return None; + } + } + Some(event_id) + }) .broad_and_then(|event_id| services.rooms.timeline.get_pdu_json(event_id)) .broad_and_then(|pdu| { services @@ -238,6 +258,17 @@ async fn create_join_event( .rooms .auth_chain .event_ids_iter(room_id, starting_events) + .broad_filter_map(|event_id| async { + if omit_members && event_id.as_ref().is_ok_and(|e| state_ids.contains(e)) { + // Don't include this event if it's already in the state + trace!( + "omitting member event {event_id:?} from returned auth chain as it is \ + already in state" + ); + return None; + } + Some(event_id) + }) .broad_and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) @@ -252,11 +283,27 @@ async fn create_join_event( .await?; services.sending.send_pdu_room(room_id, &pdu_id).await?; - - Ok(create_join_event::v1::RoomState { + let servers_in_room: Option> = if omit_members { + None + } else { + debug!("Fetching list of servers in room"); + Some( + services + .rooms + .state_cache + .room_servers(room_id) + .map(|sn| sn.as_str().to_owned()) + .collect() + .await, + ) + }; + debug!("Returning send_join data"); + Ok(create_join_event::v2::RoomState { auth_chain, state, event: to_raw_value(&CanonicalJsonValue::Object(value)).ok(), + members_omitted: omit_members, + servers_in_room, }) } @@ -294,11 +341,24 @@ pub(crate) async fn create_join_event_v1_route( } } - let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu) + info!("Providing send_join for {} in {}", body.origin(), &body.room_id); + let now = Instant::now(); + let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu, false) .boxed() .await?; + let transformed = create_join_event::v1::RoomState { + auth_chain: room_state.auth_chain, + state: room_state.state, + event: room_state.event, + }; + info!( + "Finished creating the send_join payload for {} in {} in {:?}", + body.origin(), + &body.room_id, + now.elapsed() + ); - Ok(create_join_event::v1::Response { room_state }) + Ok(create_join_event::v1::Response { room_state: transformed }) } /// # `PUT /_matrix/federation/v2/send_join/{roomId}/{eventId}` @@ -329,17 +389,18 @@ pub(crate) async fn create_join_event_v2_route( } } - let create_join_event::v1::RoomState { auth_chain, state, event } = - create_join_event(&services, body.origin(), &body.room_id, &body.pdu) + info!("Providing send_join for {} in {}", body.origin(), &body.room_id); + let now = Instant::now(); + let room_state = + create_join_event(&services, body.origin(), &body.room_id, &body.pdu, body.omit_members) .boxed() .await?; - let room_state = create_join_event::v2::RoomState { - members_omitted: false, - auth_chain, - state, - event, - servers_in_room: None, - }; + info!( + "Finished creating the send_join payload for {} in {} in {:?}", + body.origin(), + &body.room_id, + now.elapsed() + ); Ok(create_join_event::v2::Response { room_state }) } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index f7f7d043..028c3e51 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -452,7 +452,7 @@ async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result Date: Wed, 3 Sep 2025 15:08:28 +0100 Subject: [PATCH 3/6] fix(PR977): Omitting redundant entries from the auth_chain caused problems --- src/api/server/send_join.rs | 49 ++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 3aebbbe7..a52b5391 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -1,6 +1,6 @@ #![allow(deprecated)] -use std::{borrow::Borrow, time::Instant}; +use std::{borrow::Borrow, time::Instant, vec}; use axum::extract::State; use conduwuit::{ @@ -258,17 +258,17 @@ async fn create_join_event( .rooms .auth_chain .event_ids_iter(room_id, starting_events) - .broad_filter_map(|event_id| async { - if omit_members && event_id.as_ref().is_ok_and(|e| state_ids.contains(e)) { - // Don't include this event if it's already in the state - trace!( - "omitting member event {event_id:?} from returned auth chain as it is \ - already in state" - ); - return None; - } - Some(event_id) - }) + // .broad_filter_map(|event_id| async { + // if omit_members && event_id.as_ref().is_ok_and(|e| state_ids.contains(e)) { + // // Don't include this event if it's already in the state + // trace!( + // "omitting member event {event_id:?} from returned auth chain as it is \ + // already in state" + // ); + // return None; + // } + // Some(event_id) + // }) .broad_and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) @@ -283,19 +283,24 @@ async fn create_join_event( .await?; services.sending.send_pdu_room(room_id, &pdu_id).await?; - let servers_in_room: Option> = if omit_members { + let servers_in_room: Option> = if !omit_members { None } else { debug!("Fetching list of servers in room"); - Some( - services - .rooms - .state_cache - .room_servers(room_id) - .map(|sn| sn.as_str().to_owned()) - .collect() - .await, - ) + let servers: Vec = services + .rooms + .state_cache + .room_servers(room_id) + .map(|sn| sn.as_str().to_owned()) + .collect() + .await; + // If there's no servers, just add us + let servers = if servers.is_empty() { + vec![services.globals.server_name().to_string()] + } else { + servers + }; + Some(servers) }; debug!("Returning send_join data"); Ok(create_join_event::v2::RoomState { From 048157197e621845e399de00fe6fcc6c76b9540b Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 7 Sep 2025 20:23:57 +0100 Subject: [PATCH 4/6] fix(PR977): Adjust some log levels --- src/api/server/send_join.rs | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index a52b5391..3145f9fa 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -204,7 +204,7 @@ async fn create_join_event( .lock(room_id) .await; - debug!("Acquired send_join mutex, persisting join event"); + trace!("Acquired send_join mutex, persisting join event"); let pdu_id = services .rooms .event_handler @@ -214,7 +214,7 @@ async fn create_join_event( .ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?; drop(mutex_lock); - debug!("Fetching current state IDs"); + trace!("Fetching current state IDs"); let state_ids: Vec = services .rooms .state_accessor @@ -258,17 +258,6 @@ async fn create_join_event( .rooms .auth_chain .event_ids_iter(room_id, starting_events) - // .broad_filter_map(|event_id| async { - // if omit_members && event_id.as_ref().is_ok_and(|e| state_ids.contains(e)) { - // // Don't include this event if it's already in the state - // trace!( - // "omitting member event {event_id:?} from returned auth chain as it is \ - // already in state" - // ); - // return None; - // } - // Some(event_id) - // }) .broad_and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) @@ -281,12 +270,12 @@ async fn create_join_event( .try_collect() .boxed() .await?; - + info!(fast_join = %omit_members, "Sending a join for {origin} to {room_id}"); services.sending.send_pdu_room(room_id, &pdu_id).await?; let servers_in_room: Option> = if !omit_members { None } else { - debug!("Fetching list of servers in room"); + trace!("Fetching list of servers in room"); let servers: Vec = services .rooms .state_cache @@ -296,6 +285,10 @@ async fn create_join_event( .await; // If there's no servers, just add us let servers = if servers.is_empty() { + warn!( + "Failed to find any servers in {room_id}, adding our own server name as a last \ + resort" + ); vec![services.globals.server_name().to_string()] } else { servers @@ -346,7 +339,6 @@ pub(crate) async fn create_join_event_v1_route( } } - info!("Providing send_join for {} in {}", body.origin(), &body.room_id); let now = Instant::now(); let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu, false) .boxed() @@ -357,7 +349,7 @@ pub(crate) async fn create_join_event_v1_route( event: room_state.event, }; info!( - "Finished creating the send_join payload for {} in {} in {:?}", + "Finished sending a join for {} in {} in {:?}", body.origin(), &body.room_id, now.elapsed() @@ -394,14 +386,13 @@ pub(crate) async fn create_join_event_v2_route( } } - info!("Providing send_join for {} in {}", body.origin(), &body.room_id); let now = Instant::now(); let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu, body.omit_members) .boxed() .await?; info!( - "Finished creating the send_join payload for {} in {} in {:?}", + "Finished sending a join for {} in {} in {:?}", body.origin(), &body.room_id, now.elapsed() From f46cb6b15cdaed3af665614ed7372387e487815f Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 7 Sep 2025 20:26:26 +0100 Subject: [PATCH 5/6] perf(pr977): Remove redundant ACL check in send_join --- src/api/server/send_join.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 3145f9fa..aa93047b 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -56,8 +56,10 @@ async fn create_join_event( // We do not add the event_id field to the pdu here because of signature and // hashes checks + trace!("Getting room version"); let room_version_id = services.rooms.state.get_room_version(room_id).await?; + trace!("Generating event ID and converting to canonical json"); let Ok((event_id, mut value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { // Event could not be converted to canonical json return Err!(Request(BadJson("Could not convert event to canonical json."))); @@ -106,7 +108,6 @@ async fn create_join_event( ))); } - // ACL check sender user server name let sender: OwnedUserId = serde_json::from_value( value .get("sender") @@ -116,12 +117,6 @@ async fn create_join_event( ) .map_err(|e| err!(Request(BadJson(warn!("sender property is not a valid user ID: {e}")))))?; - services - .rooms - .event_handler - .acl_check(sender.server_name(), room_id) - .await?; - // check if origin server is trying to send for another server if sender.server_name() != origin { return Err!(Request(Forbidden("Not allowed to join on behalf of another server."))); From cf7da3a9b4a5a095948a29f401220acd5e5564e0 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 7 Sep 2025 20:32:26 +0100 Subject: [PATCH 6/6] feat(PR977): Log more things in the join process --- src/api/server/send_join.rs | 41 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index aa93047b..d92581a1 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -178,11 +178,6 @@ async fn create_join_event( } } - services - .server_keys - .hash_and_sign_event(&mut value, &room_version_id) - .map_err(|e| err!(Request(InvalidParam(warn!("Failed to sign send_join event: {e}")))))?; - let origin: OwnedServerName = serde_json::from_value( value .get("origin") @@ -192,6 +187,12 @@ async fn create_join_event( ) .map_err(|e| err!(Request(BadJson("Event has an invalid origin server name: {e}"))))?; + trace!("Signing send_join event"); + services + .server_keys + .hash_and_sign_event(&mut value, &room_version_id) + .map_err(|e| err!(Request(InvalidParam(warn!("Failed to sign send_join event: {e}")))))?; + let mutex_lock = services .rooms .event_handler @@ -218,21 +219,19 @@ async fn create_join_event( .collect() .await; - #[allow(clippy::unnecessary_unwrap)] + trace!(%omit_members, "Constructing current state"); let state = state_ids .iter() .try_stream() .broad_filter_map(|event_id| async move { - if omit_members && event_id.is_ok() { - let pdu = services - .rooms - .timeline - .get_pdu(event_id.as_ref().unwrap()) - .await; - if pdu.is_ok_and(|p| p.kind().to_cow_str() == "m.room.member") { - trace!("omitting member event {event_id:?} from returned state"); - // skip members - return None; + if omit_members { + if let Ok(e) = event_id.as_ref() { + let pdu = services.rooms.timeline.get_pdu(e).await; + if pdu.is_ok_and(|p| p.kind().to_cow_str() == "m.room.member") { + trace!("omitting member event {e:?} from returned state"); + // skip members + return None; + } } } Some(event_id) @@ -249,6 +248,7 @@ async fn create_join_event( .await?; let starting_events = state_ids.iter().map(Borrow::borrow); + trace!("Constructing auth chain"); let auth_chain = services .rooms .auth_chain @@ -265,8 +265,9 @@ async fn create_join_event( .try_collect() .boxed() .await?; - info!(fast_join = %omit_members, "Sending a join for {origin} to {room_id}"); + info!(fast_join = %omit_members, "Sending join event to other servers"); services.sending.send_pdu_room(room_id, &pdu_id).await?; + debug!("Finished sending join event"); let servers_in_room: Option> = if !omit_members { None } else { @@ -280,12 +281,10 @@ async fn create_join_event( .await; // If there's no servers, just add us let servers = if servers.is_empty() { - warn!( - "Failed to find any servers in {room_id}, adding our own server name as a last \ - resort" - ); + warn!("Failed to find any servers, adding our own server name as a last resort"); vec![services.globals.server_name().to_string()] } else { + trace!("Found {} servers in room", servers.len()); servers }; Some(servers)