diff --git a/src/main/java/org/geysermc/extension/connect/PacketHandler.java b/src/main/java/org/geysermc/extension/connect/PacketHandler.java
index ef948b6..4a4d9b3 100644
--- a/src/main/java/org/geysermc/extension/connect/PacketHandler.java
+++ b/src/main/java/org/geysermc/extension/connect/PacketHandler.java
@@ -68,10 +68,10 @@ public class PacketHandler extends UpstreamPacketHandler {
 
     @Override
     public void onDisconnect(String reason) {
-        if (session.getAuthData() != null) {
-            geyserConnect.logger().info(Utils.displayName(session) + " has disconnected (" + reason + ")");
-            ServerManager.unloadServers(session);
-        }
+        // The user has disconnected without having connected to an actual server. If they have connected to
+        // a server (transfer packet or geyser proxy), then the original packet handler has been restored.
+        ServerManager.unloadServers(session);
+        originalPacketHandler.onDisconnect(reason);
     }
 
     @Override
@@ -155,12 +155,12 @@ public class PacketHandler extends UpstreamPacketHandler {
 
     @Override
     public PacketSignal handle(ResourcePackClientResponsePacket packet) {
-        return originalPacketHandler.handle(packet);
+        return originalPacketHandler.handle(packet); // relies on state in the original handler
     }
 
     @Override
     public PacketSignal handle(ResourcePackChunkRequestPacket packet) {
-        return originalPacketHandler.handle(packet);
+        return originalPacketHandler.handle(packet); // relies on state in the original handler
     }
 }
 
diff --git a/src/main/java/org/geysermc/extension/connect/utils/Utils.java b/src/main/java/org/geysermc/extension/connect/utils/Utils.java
index 17feb2a..75c8390 100644
--- a/src/main/java/org/geysermc/extension/connect/utils/Utils.java
+++ b/src/main/java/org/geysermc/extension/connect/utils/Utils.java
@@ -84,6 +84,13 @@ public class Utils {
         GeyserConnect.instance().logger().info("Sending " + Utils.displayName(session) + " to " + server.title());
         GeyserConnect.instance().logger().debug(server.toString());
 
+        // Save the player's servers since we are changing packet handlers
+        // (and they are going to disconnect if it is a bedrock server)
+        ServerManager.unloadServers(session);
+
+        // Restore the original packet handler
+        session.getUpstream().getSession().setPacketHandler(originalPacketHandler);
+
         if (server.bedrock()) {
             // Send them to the bedrock server
             TransferPacket transferPacket = new TransferPacket();
@@ -91,12 +98,6 @@ public class Utils {
             transferPacket.setPort(server.port());
             session.sendUpstreamPacket(transferPacket);
         } else {
-            // Save the players servers since we are changing packet handlers
-            ServerManager.unloadServers(session);
-
-            // Restore the original packet handler
-            session.getUpstream().getSession().setPacketHandler(originalPacketHandler);
-
             // Set the remote server and un-initialize the session
             session.remoteServer(server);
             session.getUpstream().setInitialized(false);