diff --git a/src/main/java/org/geysermc/connect/MasterServer.java b/src/main/java/org/geysermc/connect/MasterServer.java
index 01840f0..43aca66 100644
--- a/src/main/java/org/geysermc/connect/MasterServer.java
+++ b/src/main/java/org/geysermc/connect/MasterServer.java
@@ -66,7 +66,7 @@ public class MasterServer {
     private final ScheduledExecutorService generalThreadPool;
 
     @Getter
-    private final Map<InetSocketAddress, Player> players = new HashMap<>();
+    private final Map<String, Player> players = new HashMap<>();
 
     @Getter
     private GeyserProxyBootstrap geyserProxy;
diff --git a/src/main/java/org/geysermc/connect/PacketHandler.java b/src/main/java/org/geysermc/connect/PacketHandler.java
index 8e83a05..9d372cf 100644
--- a/src/main/java/org/geysermc/connect/PacketHandler.java
+++ b/src/main/java/org/geysermc/connect/PacketHandler.java
@@ -137,7 +137,7 @@ public class PacketHandler implements BedrockPacketHandler {
 
                 // Create a new player and add it to the players list
                 player = new Player(extraData, session);
-                masterServer.getPlayers().put(session.getAddress(), player);
+                masterServer.getPlayers().put(player.getXuid(), player);
 
                 // Tell the client we have logged in successfully
                 PlayStatusPacket playStatusPacket = new PlayStatusPacket();
diff --git a/src/main/java/org/geysermc/connect/proxy/GeyserProxySession.java b/src/main/java/org/geysermc/connect/proxy/GeyserProxySession.java
index c4d0357..b819c9d 100644
--- a/src/main/java/org/geysermc/connect/proxy/GeyserProxySession.java
+++ b/src/main/java/org/geysermc/connect/proxy/GeyserProxySession.java
@@ -31,6 +31,7 @@ import org.geysermc.connector.GeyserConnector;
 import org.geysermc.connector.network.session.GeyserSession;
 import org.geysermc.connect.MasterServer;
 import org.geysermc.connect.utils.Player;
+import org.geysermc.connector.network.session.auth.AuthData;
 
 public class GeyserProxySession extends GeyserSession {
 
@@ -43,9 +44,10 @@ public class GeyserProxySession extends GeyserSession {
         this.bedrockServerSession = bedrockServerSession;
     }
 
+    @Override
     public void authenticate(String username, String password) {
         // Get the player based on the connection address
-        Player player = MasterServer.getInstance().getPlayers().get(bedrockServerSession.getAddress());
+        Player player = MasterServer.getInstance().getPlayers().get(getAuthData().getXboxUUID());
         if (player != null && player.getCurrentServer() != null) {
             // Set the remote server info for the player
             connector.getRemoteServer().setAddress(player.getCurrentServer().getAddress());
@@ -56,4 +58,15 @@ public class GeyserProxySession extends GeyserSession {
             bedrockServerSession.disconnect("Please connect to the master server and pick a server first!");
         }
     }
+
+    @Override
+    public void setAuthenticationData(AuthData authData) {
+        super.setAuthenticationData(authData);
+
+        Player player = MasterServer.getInstance().getPlayers().get(authData.getXboxUUID());
+        if (player == null) {
+            bedrockServerSession.disconnect("Please connect to the master server and pick a server first!");
+            return;
+        }
+    }
 }
diff --git a/src/main/java/org/geysermc/connect/proxy/ProxyConnectorServerEventHandler.java b/src/main/java/org/geysermc/connect/proxy/ProxyConnectorServerEventHandler.java
index 3e6ea42..ccabf7b 100644
--- a/src/main/java/org/geysermc/connect/proxy/ProxyConnectorServerEventHandler.java
+++ b/src/main/java/org/geysermc/connect/proxy/ProxyConnectorServerEventHandler.java
@@ -45,23 +45,19 @@ public class ProxyConnectorServerEventHandler extends ConnectorServerEventHandle
 
     @Override
     public void onSessionCreation(BedrockServerSession bedrockServerSession) {
-        bedrockServerSession.setPacketCodec(GeyserConnector.BEDROCK_PACKET_CODEC); // Only done here as it allows us to disconnect the player
-
-        Player player = MasterServer.getInstance().getPlayers().get(bedrockServerSession.getAddress());
-        if (player == null) {
-            bedrockServerSession.disconnect("Please connect to the master server and pick a server first!");
-            return;
-        }
-
         super.onSessionCreation(bedrockServerSession);
 
         // This doesn't clean up the old packet handler, so may cause a memory leak?
-        bedrockServerSession.setPacketHandler(new UpstreamPacketHandler(connector, new GeyserProxySession(connector, bedrockServerSession)));
+        GeyserProxySession session = new GeyserProxySession(connector, bedrockServerSession);
+        bedrockServerSession.setPacketHandler(new UpstreamPacketHandler(connector, session));
 
         // Add another disconnect handler to remove the player on final disconnect
         bedrockServerSession.addDisconnectHandler(disconnectReason -> {
-            MasterServer.getInstance().getLogger().debug("Player disconnected from geyser proxy: " + player.getDisplayName() + " (" + disconnectReason + ")");
-            MasterServer.getInstance().getPlayers().remove(bedrockServerSession.getAddress());
+            Player player = MasterServer.getInstance().getPlayers().get(session.getAuthData().getXboxUUID());
+            if (player != null) {
+                MasterServer.getInstance().getLogger().debug("Player disconnected from Geyser proxy: " + player.getDisplayName() + " (" + disconnectReason + ")");
+                MasterServer.getInstance().getPlayers().remove(session.getAuthData().getXboxUUID());
+            }
         });
     }
 }