diff --git a/src/xrt/compositor/client/comp_d3d11_client.cpp b/src/xrt/compositor/client/comp_d3d11_client.cpp
index 715daaf19..cfcccebc8 100644
--- a/src/xrt/compositor/client/comp_d3d11_client.cpp
+++ b/src/xrt/compositor/client/comp_d3d11_client.cpp
@@ -776,10 +776,14 @@ client_d3d11_compositor_init_try_timeline_semaphores(struct client_d3d11_composi
 	}
 	D3D_INFO(c, "Native compositor created a timeline semaphore for us.");
 
+	// Because importFence throws on failure we use this ref.
 	unique_compositor_semaphore_ref timeline_semaphore{xcsem};
 
+	// unique_compositor_semaphore_ref now owns the handle.
+	HANDLE timeline_semaphore_handle_raw = timeline_semaphore_handle.release();
+
 	// try to import and signal
-	wil::com_ptr<ID3D11Fence> fence = import_fence(*(c->fence_device), timeline_semaphore_handle.get());
+	wil::com_ptr<ID3D11Fence> fence = import_fence(*(c->fence_device), timeline_semaphore_handle_raw);
 	HRESULT hr = c->fence_context->Signal(fence.get(), c->timeline_semaphore_value);
 	if (!SUCCEEDED(hr)) {
 		D3D_WARN(c,
diff --git a/src/xrt/compositor/client/comp_d3d12_client.cpp b/src/xrt/compositor/client/comp_d3d12_client.cpp
index 1644be871..ba5611fd6 100644
--- a/src/xrt/compositor/client/comp_d3d12_client.cpp
+++ b/src/xrt/compositor/client/comp_d3d12_client.cpp
@@ -1040,13 +1040,13 @@ client_d3d12_compositor_init_try_timeline_semaphores(struct client_d3d12_composi
 	// Because importFence throws on failure we use this ref.
 	unique_compositor_semaphore_ref timeline_semaphore{xcsem};
 
+	// unique_compositor_semaphore_ref now owns the handle.
+	HANDLE timeline_semaphore_handle_raw = timeline_semaphore_handle.release();
+
 	// Try to import, importFence throws on failure.
 	wil::com_ptr<ID3D12Fence1> fence = xrt::auxiliary::d3d::d3d12::importFence( //
 	    *(c->device),                                                           //
-	    timeline_semaphore_handle.get());                                       //
-
-	// The fence now owns the handle., importFence throws on failure.
-	timeline_semaphore_handle.release();
+	    timeline_semaphore_handle_raw);                                         //
 
 	// Check flags.
 	D3D12_FENCE_FLAGS flags = fence->GetCreationFlags();