diff --git a/wezterm-ssh/tests/sshd.rs b/wezterm-ssh/tests/sshd.rs
index 69c48b5a0..80e709ce3 100644
--- a/wezterm-ssh/tests/sshd.rs
+++ b/wezterm-ssh/tests/sshd.rs
@@ -3,7 +3,7 @@ use assert_fs::TempDir;
 use rstest::*;
 use std::collections::HashMap;
 use std::io::Result as IoResult;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::{Child, Command};
 use std::sync::LazyLock;
 use std::time::Duration;
@@ -59,37 +59,24 @@ impl SshKeygen {
     }
 }
 
-pub struct SshAgent;
+pub struct SshAgent {
+    child: Child,
+}
 
-impl SshAgent {
-    pub fn generate_shell_env() -> IoResult<HashMap<String, String>> {
-        let output = Command::new("ssh-agent").arg("-s").output()?;
-        let stdout = String::from_utf8(output.stdout)
-            .map_err(|x| std::io::Error::new(std::io::ErrorKind::InvalidData, x))?;
-        Ok(stdout
-            .split(";")
-            .map(str::trim)
-            .filter(|s| s.contains("="))
-            .map(|s| {
-                let mut tokens = s.split("=");
-                let key = tokens.next().unwrap().trim().to_string();
-                let rest = tokens
-                    .map(str::trim)
-                    .map(ToString::to_string)
-                    .collect::<Vec<String>>()
-                    .join("=");
-                (key, rest)
-            })
-            .collect::<HashMap<String, String>>())
+impl Drop for SshAgent {
+    fn drop(&mut self) {
+        self.child.kill().ok();
     }
+}
 
-    pub fn update_tests_with_shell_env() -> IoResult<()> {
-        let env_map = Self::generate_shell_env()?;
-        for (key, value) in env_map {
-            std::env::set_var(key, value);
-        }
-
-        Ok(())
+impl SshAgent {
+    fn with_sock(path: &Path) -> IoResult<Self> {
+        let child = Command::new("ssh-agent")
+            .arg("-a")
+            .arg(path)
+            .arg("-D")
+            .spawn()?;
+        Ok(Self { child })
     }
 }
 
@@ -261,6 +248,9 @@ pub struct Sshd {
 
     /// Temporary directory used to hold resources for sshd such as its config, keys, and log
     pub tmp: TempDir,
+
+    agent_sock: PathBuf,
+    _agent: SshAgent,
 }
 
 impl Sshd {
@@ -272,8 +262,11 @@ impl Sshd {
 
         let tmp = TempDir::new()?;
 
-        // Ensure that everything needed for interacting with ssh-agent is set
-        SshAgent::update_tests_with_shell_env()?;
+        // Spawn an agent. It doesn't strictly belong to the daemon in normal
+        // operation, but we take care of it together here in the test context
+        // because this is where we know the temp dir for the test-specific fixture
+        let agent_sock = tmp.join("agent.sock");
+        let agent = SshAgent::with_sock(&agent_sock)?;
 
         // ssh-keygen -t rsa -f $ROOT/id_rsa -N "" -q
         let id_rsa_file = tmp.child("id_rsa");
@@ -313,7 +306,13 @@ impl Sshd {
         let (child, port) = Self::try_spawn_next(sshd_config_file.path(), sshd_log_file.path())
             .expect("No open port available for sshd");
 
-        Ok(Self { child, port, tmp })
+        Ok(Self {
+            child,
+            port,
+            tmp,
+            _agent: agent,
+            agent_sock,
+        })
     }
 
     fn try_spawn_next(
@@ -453,6 +452,11 @@ pub async fn session(#[default(Config::new())] config: Config, sshd: Sshd) -> Se
     #[cfg(not(feature = "libssh-rs"))]
     config.insert("wezterm_ssh_backend".to_string(), "ssh2".to_string());
 
+    config.insert(
+        "identityagent".to_string(),
+        format!("{}", sshd.agent_sock.display()),
+    );
+
     config.insert("user".to_string(), USERNAME.to_string());
     config.insert("identitiesonly".to_string(), "yes".to_string());
     config.insert(