From ef0f552c3d773a2a4a04236c5d5ca78e3f90aad4 Mon Sep 17 00:00:00 2001 From: Carl Philipp Klemm Date: Wed, 8 Oct 2025 21:51:36 +0200 Subject: [PATCH] c/compositor: support compensation for rolling scanout hmds Currently only the compute renderer performs compensation Currenlty only a top to bottom scanout direction is compensated adds information on scanout to xrt_hmd_parts Signed-off-by: Carl Philipp Klemm Part-of: --- src/xrt/compositor/main/comp_renderer.c | 127 ++++-- src/xrt/compositor/render/render_compute.c | 399 +++++++++--------- src/xrt/compositor/render/render_interface.h | 44 +- src/xrt/compositor/render/render_util.c | 11 + src/xrt/compositor/shaders/distortion.comp | 29 +- .../compositor/util/comp_high_level_render.c | 20 +- .../compositor/util/comp_high_level_render.h | 19 +- src/xrt/compositor/util/comp_render.h | 29 +- src/xrt/compositor/util/comp_render_cs.c | 220 +++++----- src/xrt/compositor/util/comp_render_gfx.c | 14 +- src/xrt/drivers/android/android_sensors.c | 2 + src/xrt/drivers/blubur_s1/blubur_s1_hmd.c | 2 + src/xrt/drivers/hdk/hdk_device.cpp | 3 + src/xrt/drivers/ohmd/oh_device.c | 2 + src/xrt/drivers/rift/rift_hmd.c | 2 + src/xrt/drivers/rokid/rokid_hmd.c | 2 + src/xrt/drivers/sample/sample_hmd.c | 2 + src/xrt/drivers/simula/svr_hmd.c | 2 + src/xrt/drivers/steamvr_lh/device.cpp | 28 +- src/xrt/drivers/steamvr_lh/device.hpp | 3 + src/xrt/drivers/survive/survive_driver.c | 10 + src/xrt/drivers/vive/vive_device.c | 10 + src/xrt/drivers/xreal_air/xreal_air_hmd.c | 1 + src/xrt/include/xrt/xrt_defines.h | 14 + src/xrt/include/xrt/xrt_device.h | 2 + src/xrt/targets/sdl_test/sdl_device.c | 1 + 26 files changed, 596 insertions(+), 402 deletions(-) diff --git a/src/xrt/compositor/main/comp_renderer.c b/src/xrt/compositor/main/comp_renderer.c index 9cc7b63cc..2c83d865d 100644 --- a/src/xrt/compositor/main/comp_renderer.c +++ b/src/xrt/compositor/main/comp_renderer.c @@ -242,7 +242,8 @@ static void calc_pose_data(struct comp_renderer *r, enum comp_target_fov_source fov_source, struct xrt_fov out_fovs[XRT_MAX_VIEWS], - struct xrt_pose out_world[XRT_MAX_VIEWS], + struct xrt_pose out_world_scanout_begin[XRT_MAX_VIEWS], + struct xrt_pose out_world_scanout_end[XRT_MAX_VIEWS], struct xrt_pose out_eye[XRT_MAX_VIEWS], uint32_t view_count) { @@ -254,18 +255,29 @@ calc_pose_data(struct comp_renderer *r, 0.0f, }; - struct xrt_space_relation head_relation = XRT_SPACE_RELATION_ZERO; + struct xrt_space_relation head_relation[2] = XRT_SPACE_RELATION_ZERO; struct xrt_fov xdev_fovs[XRT_MAX_VIEWS] = XRT_STRUCT_INIT; - struct xrt_pose xdev_poses[XRT_MAX_VIEWS] = XRT_STRUCT_INIT; - - xrt_result_t xret = xrt_device_get_view_poses( // - r->c->xdev, // - &default_eye_relation, // - r->c->frame.rendering.predicted_display_time_ns, // at_timestamp_ns - view_count, // - &head_relation, // out_head_relation - xdev_fovs, // out_fovs - xdev_poses); // out_poses + struct xrt_pose xdev_poses[2][XRT_MAX_VIEWS] = XRT_STRUCT_INIT; + + uint64_t scanout_time_ns = 0; + if (r->c->xdev->hmd->screens[0].scanout_direction == XRT_SCANOUT_DIRECTION_TOP_TO_BOTTOM) { + scanout_time_ns = r->c->xdev->hmd->screens[0].scanout_time_ns; + } else if (r->c->xdev->hmd->screens[0].scanout_direction != XRT_SCANOUT_DIRECTION_NONE) { + COMP_SPEW(r->c, "Unable to apply scanout compensation as only DIRECTION_TOP_TO_BOTTOM is supported"); + } + + int64_t begin_timestamp_ns = r->c->frame.rendering.predicted_display_time_ns; + int64_t end_timestamp_ns = begin_timestamp_ns + scanout_time_ns; + + // Pose at beginning of scanout + xrt_result_t xret = xrt_device_get_view_poses( // + r->c->xdev, // + &default_eye_relation, // + begin_timestamp_ns, // at_timestamp_ns + view_count, // + &head_relation[0], // out_head_relation + xdev_fovs, // out_fovs + xdev_poses[0]); if (xret != XRT_SUCCESS) { struct u_pp_sink_stack_only sink; u_pp_delegate_t dg = u_pp_sink_stack_only_init(&sink); @@ -274,6 +286,30 @@ calc_pose_data(struct comp_renderer *r, return; } + // Pose at end of scanout + if (scanout_time_ns != 0) { + xret = xrt_device_get_view_poses( // + r->c->xdev, // + &default_eye_relation, // + end_timestamp_ns, // at_timestamp_ns + view_count, // + &head_relation[1], // out_head_relation + xdev_fovs, // out_fovs + xdev_poses[1]); // out_poses + if (xret != XRT_SUCCESS) { + struct u_pp_sink_stack_only sink; + u_pp_delegate_t dg = u_pp_sink_stack_only_init(&sink); + u_pp_xrt_result(dg, xret); + U_LOG_E("xrt_device_get_view_poses failed: %s", sink.buffer); + return; + } + } else { + for (size_t i = 0; i < XRT_MAX_VIEWS; ++i) { + xdev_poses[1][i] = xdev_poses[0][i]; + } + head_relation[1] = head_relation[0]; + } + struct xrt_fov dist_fov[XRT_MAX_VIEWS] = XRT_STRUCT_INIT; for (uint32_t i = 0; i < view_count; i++) { dist_fov[i] = r->c->xdev->hmd->distortion.fov[i]; @@ -288,22 +324,32 @@ calc_pose_data(struct comp_renderer *r, for (uint32_t i = 0; i < view_count; i++) { const struct xrt_fov fov = use_xdev ? xdev_fovs[i] : dist_fov[i]; - const struct xrt_pose eye_pose = xdev_poses[i]; + const struct xrt_pose eye_pose_scanout_start = xdev_poses[0][i]; + const struct xrt_pose eye_pose_scanout_end = xdev_poses[1][i]; - struct xrt_space_relation result = {0}; + struct xrt_space_relation result_scanout_start = {0}; + struct xrt_space_relation result_scanout_end = {0}; struct xrt_relation_chain xrc = {0}; - m_relation_chain_push_pose_if_not_identity(&xrc, &eye_pose); - m_relation_chain_push_relation(&xrc, &head_relation); - m_relation_chain_resolve(&xrc, &result); + + m_relation_chain_push_pose_if_not_identity(&xrc, &eye_pose_scanout_start); + m_relation_chain_push_relation(&xrc, &head_relation[0]); + m_relation_chain_resolve(&xrc, &result_scanout_start); + + xrc = (struct xrt_relation_chain){0}; + + m_relation_chain_push_pose_if_not_identity(&xrc, &eye_pose_scanout_end); + m_relation_chain_push_relation(&xrc, &head_relation[1]); + m_relation_chain_resolve(&xrc, &result_scanout_end); // Results to callers. out_fovs[i] = fov; - out_world[i] = result.pose; - out_eye[i] = eye_pose; + out_world_scanout_begin[i] = result_scanout_start.pose; + out_world_scanout_end[i] = result_scanout_end.pose; + out_eye[i] = eye_pose_scanout_start; // For remote rendering targets. r->c->base.frame_params.fovs[i] = fov; - r->c->base.frame_params.poses[i] = result.pose; + r->c->base.frame_params.poses[i] = result_scanout_start.pose; } } @@ -866,15 +912,17 @@ dispatch_graphics(struct comp_renderer *r, // Device view information. struct xrt_fov fovs[XRT_MAX_VIEWS]; - struct xrt_pose world_poses[XRT_MAX_VIEWS]; + struct xrt_pose world_poses_scanout_begin[XRT_MAX_VIEWS]; + struct xrt_pose world_poses_scanout_end[XRT_MAX_VIEWS]; struct xrt_pose eye_poses[XRT_MAX_VIEWS]; - calc_pose_data( // - r, // - fov_source, // - fovs, // - world_poses, // - eye_poses, // - render->r->view_count); // + calc_pose_data( // + r, // + fov_source, // + fovs, // + world_poses_scanout_begin, // + world_poses_scanout_end, // + eye_poses, // + render->r->view_count); // // Does everything. chl_frame_state_gfx_default_pipeline( // @@ -882,7 +930,7 @@ dispatch_graphics(struct comp_renderer *r, render, // layers, // layer_count, // - world_poses, // + world_poses_scanout_begin, // eye_poses, // fovs, // rtr, // @@ -924,15 +972,17 @@ dispatch_compute(struct comp_renderer *r, // Device view information. struct xrt_fov fovs[XRT_MAX_VIEWS]; - struct xrt_pose world_poses[XRT_MAX_VIEWS]; + struct xrt_pose world_poses_scanout_begin[XRT_MAX_VIEWS]; + struct xrt_pose world_poses_scanout_end[XRT_MAX_VIEWS]; struct xrt_pose eye_poses[XRT_MAX_VIEWS]; - calc_pose_data( // - r, // - fov_source, // - fovs, // - world_poses, // - eye_poses, // - render->r->view_count); // + calc_pose_data( // + r, // + fov_source, // + fovs, // + world_poses_scanout_begin, // + world_poses_scanout_end, // + eye_poses, // + render->r->view_count); // // Target Vulkan resources.. VkImage target_image = r->c->target->images[r->acquired_buffer].handle; @@ -948,7 +998,8 @@ dispatch_compute(struct comp_renderer *r, render, // layers, // layer_count, // - world_poses, // + world_poses_scanout_begin, // + world_poses_scanout_end, // eye_poses, // fovs, // target_image, // diff --git a/src/xrt/compositor/render/render_compute.c b/src/xrt/compositor/render/render_compute.c index 27b1953cf..62d7138b9 100644 --- a/src/xrt/compositor/render/render_compute.c +++ b/src/xrt/compositor/render/render_compute.c @@ -278,6 +278,129 @@ update_compute_descriptor_set_target(struct vk_bundle *vk, NULL); // pDescriptorCopies } +static void +dispatch_project_pipeline(struct render_compute *render, + VkSampler src_samplers[XRT_MAX_VIEWS], + VkImageView src_image_views[XRT_MAX_VIEWS], + const struct xrt_normalized_rect src_norm_rects[XRT_MAX_VIEWS], + VkImage target_image, + VkImageView target_image_view, + const struct render_viewport_data views[XRT_MAX_VIEWS], + VkPipeline pipeline) +{ + struct vk_bundle *vk = vk_from_render(render); + struct render_resources *r = render->r; + + + /* + * UBO + */ + + struct render_compute_distortion_ubo_data *data = + (struct render_compute_distortion_ubo_data *)r->compute.distortion.ubo.mapped; + for (uint32_t i = 0; i < render->r->view_count; ++i) { + data->views[i] = views[i]; + data->post_transforms[i] = src_norm_rects[i]; + } + + + /* + * Source, target and distortion images. + */ + + VkImageSubresourceRange subresource_range = { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }; + + vk_cmd_image_barrier_gpu_locked( // + vk, // + r->cmd, // + target_image, // + 0, // + VK_ACCESS_SHADER_WRITE_BIT, // + VK_IMAGE_LAYOUT_UNDEFINED, // + VK_IMAGE_LAYOUT_GENERAL, // + subresource_range); // + + VkSampler sampler = r->samplers.clamp_to_edge; + VkSampler distortion_samplers[3 * XRT_MAX_VIEWS]; + for (uint32_t i = 0; i < render->r->view_count; ++i) { + distortion_samplers[3 * i + 0] = sampler; + distortion_samplers[3 * i + 1] = sampler; + distortion_samplers[3 * i + 2] = sampler; + } + + update_compute_shared_descriptor_set( // + vk, // + r->compute.src_binding, // + src_samplers, // + src_image_views, // + r->compute.distortion_binding, // + distortion_samplers, // + r->distortion.image_views, // + r->compute.target_binding, // + target_image_view, // + r->compute.ubo_binding, // + r->compute.distortion.ubo.buffer, // + VK_WHOLE_SIZE, // + render->shared_descriptor_set, // + render->r->view_count); // + + vk->vkCmdBindPipeline( // + r->cmd, // + VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint + pipeline); // pipeline + + vk->vkCmdBindDescriptorSets( // + r->cmd, // + VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint + r->compute.distortion.pipeline_layout, // layout + 0, // firstSet + 1, // descriptorSetCount + &render->shared_descriptor_set, // pDescriptorSets + 0, // dynamicOffsetCount + NULL); // pDynamicOffsets + + + uint32_t w = 0, h = 0; + calc_dispatch_dims_views(views, render->r->view_count, &w, &h); + assert(w != 0 && h != 0); + + vk->vkCmdDispatch( // + r->cmd, // + w, // groupCountX + h, // groupCountY + 2); // groupCountZ + + VkImageMemoryBarrier memoryBarrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = target_image, + .subresourceRange = subresource_range, + }; + + vk->vkCmdPipelineBarrier( // + r->cmd, // + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // + 0, // + 0, // + NULL, // + 0, // + NULL, // + 1, // + &memoryBarrier); // +} + /* * @@ -457,14 +580,13 @@ render_compute_projection_timewarp(struct render_compute *render, const struct xrt_normalized_rect src_norm_rects[XRT_MAX_VIEWS], const struct xrt_pose src_poses[XRT_MAX_VIEWS], const struct xrt_fov src_fovs[XRT_MAX_VIEWS], - const struct xrt_pose new_poses[XRT_MAX_VIEWS], + const struct xrt_pose new_poses_scanout_begin[XRT_MAX_VIEWS], + const struct xrt_pose new_poses_scanout_end[XRT_MAX_VIEWS], VkImage target_image, VkImageView target_image_view, const struct render_viewport_data views[XRT_MAX_VIEWS]) { assert(render->r != NULL); - - struct vk_bundle *vk = vk_from_render(render); struct render_resources *r = render->r; @@ -472,13 +594,20 @@ render_compute_projection_timewarp(struct render_compute *render, * UBO */ - struct xrt_matrix_4x4 time_warp_matrix[XRT_MAX_VIEWS]; + struct xrt_matrix_4x4 time_warp_matrix_scanout_begin[XRT_MAX_VIEWS]; + struct xrt_matrix_4x4 time_warp_matrix_scanout_end[XRT_MAX_VIEWS]; for (uint32_t i = 0; i < render->r->view_count; ++i) { - render_calc_time_warp_matrix( // - &src_poses[i], // - &src_fovs[i], // - &new_poses[i], // - &time_warp_matrix[i]); // + render_calc_time_warp_matrix( // + &src_poses[i], // + &src_fovs[i], // + &new_poses_scanout_begin[i], // + &time_warp_matrix_scanout_begin[i]); // + + render_calc_time_warp_matrix( // + &src_poses[i], // + &src_fovs[i], // + &new_poses_scanout_end[i], // + &time_warp_matrix_scanout_end[i]); // } struct render_compute_distortion_ubo_data *data = @@ -486,119 +615,34 @@ render_compute_projection_timewarp(struct render_compute *render, for (uint32_t i = 0; i < render->r->view_count; ++i) { data->views[i] = views[i]; data->pre_transforms[i] = r->distortion.uv_to_tanangle[i]; - data->transforms[i] = time_warp_matrix[i]; + data->transform_timewarp_scanout_begin[i] = time_warp_matrix_scanout_begin[i]; + data->transform_timewarp_scanout_end[i] = time_warp_matrix_scanout_end[i]; data->post_transforms[i] = src_norm_rects[i]; } - /* - * Source, target and distortion images. - */ - - VkImageSubresourceRange subresource_range = { - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .baseMipLevel = 0, - .levelCount = VK_REMAINING_MIP_LEVELS, - .baseArrayLayer = 0, - .layerCount = VK_REMAINING_ARRAY_LAYERS, - }; - - vk_cmd_image_barrier_gpu_locked( // - vk, // - r->cmd, // - target_image, // - 0, // - VK_ACCESS_SHADER_WRITE_BIT, // - VK_IMAGE_LAYOUT_UNDEFINED, // - VK_IMAGE_LAYOUT_GENERAL, // - subresource_range); // - - VkSampler sampler = r->samplers.clamp_to_edge; - VkSampler distortion_samplers[3 * XRT_MAX_VIEWS]; - for (uint32_t i = 0; i < render->r->view_count; ++i) { - distortion_samplers[3 * i + 0] = sampler; - distortion_samplers[3 * i + 1] = sampler; - distortion_samplers[3 * i + 2] = sampler; - } - - update_compute_shared_descriptor_set( // - vk, // - r->compute.src_binding, // - src_samplers, // - src_image_views, // - r->compute.distortion_binding, // - distortion_samplers, // - r->distortion.image_views, // - r->compute.target_binding, // - target_image_view, // - r->compute.ubo_binding, // - r->compute.distortion.ubo.buffer, // - VK_WHOLE_SIZE, // - render->shared_descriptor_set, // - render->r->view_count); // - - vk->vkCmdBindPipeline( // - r->cmd, // - VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint - r->compute.distortion.timewarp_pipeline); // pipeline - - vk->vkCmdBindDescriptorSets( // - r->cmd, // - VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint - r->compute.distortion.pipeline_layout, // layout - 0, // firstSet - 1, // descriptorSetCount - &render->shared_descriptor_set, // pDescriptorSets - 0, // dynamicOffsetCount - NULL); // pDynamicOffsets - - - uint32_t w = 0, h = 0; - calc_dispatch_dims_views(views, render->r->view_count, &w, &h); - assert(w != 0 && h != 0); - - vk->vkCmdDispatch( // - r->cmd, // - w, // groupCountX - h, // groupCountY - 2); // groupCountZ - - VkImageMemoryBarrier memoryBarrier = { - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, - .oldLayout = VK_IMAGE_LAYOUT_GENERAL, - .newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = target_image, - .subresourceRange = subresource_range, - }; - - vk->vkCmdPipelineBarrier( // - r->cmd, // - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // - VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // - 0, // - 0, // - NULL, // - 0, // - NULL, // - 1, // - &memoryBarrier); // + dispatch_project_pipeline(render, src_samplers, src_image_views, src_norm_rects, target_image, + target_image_view, views, r->compute.distortion.timewarp_pipeline); } + +/* + * This function is intended to be used on content already timewarped to new_poses_scanout_begin. + * It performs only the timewarp nesscary to compensate for the time delta between the start and end of + * scanout. + */ void -render_compute_projection(struct render_compute *render, - VkSampler src_samplers[XRT_MAX_VIEWS], - VkImageView src_image_views[XRT_MAX_VIEWS], - const struct xrt_normalized_rect src_norm_rects[XRT_MAX_VIEWS], - VkImage target_image, - VkImageView target_image_view, - const struct render_viewport_data views[XRT_MAX_VIEWS]) +render_compute_projection_scanout_compensation(struct render_compute *render, + VkSampler src_samplers[XRT_MAX_VIEWS], + VkImageView src_image_views[XRT_MAX_VIEWS], + const struct xrt_normalized_rect src_rects[XRT_MAX_VIEWS], + const struct xrt_fov src_fovs[XRT_MAX_VIEWS], + const struct xrt_pose new_poses_scanout_begin[XRT_MAX_VIEWS], + const struct xrt_pose new_poses_scanout_end[XRT_MAX_VIEWS], + VkImage target_image, + VkImageView target_image_view, + const struct render_viewport_data views[XRT_MAX_VIEWS]) { assert(render->r != NULL); - - struct vk_bundle *vk = vk_from_render(render); struct render_resources *r = render->r; @@ -606,109 +650,46 @@ render_compute_projection(struct render_compute *render, * UBO */ - struct render_compute_distortion_ubo_data *data = - (struct render_compute_distortion_ubo_data *)r->compute.distortion.ubo.mapped; + struct xrt_matrix_4x4 time_warp_matrix_scanout_begin[XRT_MAX_VIEWS]; + struct xrt_matrix_4x4 time_warp_matrix_scanout_end[XRT_MAX_VIEWS]; for (uint32_t i = 0; i < render->r->view_count; ++i) { - data->views[i] = views[i]; - data->post_transforms[i] = src_norm_rects[i]; - } - - - /* - * Source, target and distortion images. - */ + render_calc_time_warp_projection(&src_fovs[i], &time_warp_matrix_scanout_begin[i]); - VkImageSubresourceRange subresource_range = { - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .baseMipLevel = 0, - .levelCount = VK_REMAINING_MIP_LEVELS, - .baseArrayLayer = 0, - .layerCount = VK_REMAINING_ARRAY_LAYERS, - }; - - vk_cmd_image_barrier_gpu_locked( // - vk, // - r->cmd, // - target_image, // - 0, // - VK_ACCESS_SHADER_WRITE_BIT, // - VK_IMAGE_LAYOUT_UNDEFINED, // - VK_IMAGE_LAYOUT_GENERAL, // - subresource_range); // + render_calc_time_warp_matrix( // + &new_poses_scanout_begin[i], // + &src_fovs[i], // + &new_poses_scanout_end[i], // + &time_warp_matrix_scanout_end[i]); // + } - VkSampler sampler = r->samplers.clamp_to_edge; - VkSampler distortion_samplers[3 * XRT_MAX_VIEWS]; + struct render_compute_distortion_ubo_data *data = + (struct render_compute_distortion_ubo_data *)r->compute.distortion.ubo.mapped; for (uint32_t i = 0; i < render->r->view_count; ++i) { - distortion_samplers[3 * i + 0] = sampler; - distortion_samplers[3 * i + 1] = sampler; - distortion_samplers[3 * i + 2] = sampler; + data->views[i] = views[i]; + data->pre_transforms[i] = r->distortion.uv_to_tanangle[i]; + data->transform_timewarp_scanout_begin[i] = time_warp_matrix_scanout_begin[i]; + data->transform_timewarp_scanout_end[i] = time_warp_matrix_scanout_end[i]; + data->post_transforms[i] = src_rects[i]; } - update_compute_shared_descriptor_set( // - vk, // - r->compute.src_binding, // - src_samplers, // - src_image_views, // - r->compute.distortion_binding, // - distortion_samplers, // - r->distortion.image_views, // - r->compute.target_binding, // - target_image_view, // - r->compute.ubo_binding, // - r->compute.distortion.ubo.buffer, // - VK_WHOLE_SIZE, // - render->shared_descriptor_set, // - render->r->view_count); // - - vk->vkCmdBindPipeline( // - r->cmd, // - VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint - r->compute.distortion.pipeline); // pipeline - - vk->vkCmdBindDescriptorSets( // - r->cmd, // - VK_PIPELINE_BIND_POINT_COMPUTE, // pipelineBindPoint - r->compute.distortion.pipeline_layout, // layout - 0, // firstSet - 1, // descriptorSetCount - &render->shared_descriptor_set, // pDescriptorSets - 0, // dynamicOffsetCount - NULL); // pDynamicOffsets - - - uint32_t w = 0, h = 0; - calc_dispatch_dims_views(views, render->r->view_count, &w, &h); - assert(w != 0 && h != 0); - - vk->vkCmdDispatch( // - r->cmd, // - w, // groupCountX - h, // groupCountY - 2); // groupCountZ + dispatch_project_pipeline(render, src_samplers, src_image_views, src_rects, target_image, target_image_view, + views, r->compute.distortion.timewarp_pipeline); +} - VkImageMemoryBarrier memoryBarrier = { - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, - .oldLayout = VK_IMAGE_LAYOUT_GENERAL, - .newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = target_image, - .subresourceRange = subresource_range, - }; +void +render_compute_projection_no_timewarp(struct render_compute *render, + VkSampler src_samplers[XRT_MAX_VIEWS], + VkImageView src_image_views[XRT_MAX_VIEWS], + const struct xrt_normalized_rect src_rects[XRT_MAX_VIEWS], + VkImage target_image, + VkImageView target_image_view, + const struct render_viewport_data views[XRT_MAX_VIEWS]) +{ + assert(render->r != NULL); + struct render_resources *r = render->r; - vk->vkCmdPipelineBarrier( // - r->cmd, // - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, // - VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, // - 0, // - 0, // - NULL, // - 0, // - NULL, // - 1, // - &memoryBarrier); // + dispatch_project_pipeline(render, src_samplers, src_image_views, src_rects, target_image, target_image_view, + views, r->compute.distortion.pipeline); } void diff --git a/src/xrt/compositor/render/render_interface.h b/src/xrt/compositor/render/render_interface.h index d5d2bc4cb..bcc7e30ce 100644 --- a/src/xrt/compositor/render/render_interface.h +++ b/src/xrt/compositor/render/render_interface.h @@ -94,6 +94,12 @@ extern "C" { * */ +/*! + * Create a simplified projection matrix for timewarp. + */ +void +render_calc_time_warp_projection(const struct xrt_fov *fov, struct xrt_matrix_4x4 *result); + /*! * Calculates a timewarp matrix which takes in NDC coords and gives out results * in [-1, 1] space that needs a perspective divide. @@ -1256,8 +1262,7 @@ struct render_compute_layer_ubo_data */ //! Timewarp matrices - struct xrt_matrix_4x4 transforms[RENDER_MAX_LAYERS]; - + struct xrt_matrix_4x4 transforms_timewarp[RENDER_MAX_LAYERS]; /*! * For quad layers @@ -1294,7 +1299,8 @@ struct render_compute_distortion_ubo_data struct render_viewport_data views[XRT_MAX_VIEWS]; struct xrt_normalized_rect pre_transforms[XRT_MAX_VIEWS]; struct xrt_normalized_rect post_transforms[XRT_MAX_VIEWS]; - struct xrt_matrix_4x4 transforms[XRT_MAX_VIEWS]; + struct xrt_matrix_4x4 transform_timewarp_scanout_begin[XRT_MAX_VIEWS]; + struct xrt_matrix_4x4 transform_timewarp_scanout_end[XRT_MAX_VIEWS]; }; /*! @@ -1364,7 +1370,8 @@ render_compute_projection_timewarp(struct render_compute *render, const struct xrt_normalized_rect src_rects[XRT_MAX_VIEWS], const struct xrt_pose src_poses[XRT_MAX_VIEWS], const struct xrt_fov src_fovs[XRT_MAX_VIEWS], - const struct xrt_pose new_poses[XRT_MAX_VIEWS], + const struct xrt_pose new_poses_scanout_begin[XRT_MAX_VIEWS], + const struct xrt_pose new_poses_scanout_end[XRT_MAX_VIEWS], VkImage target_image, VkImageView target_image_view, const struct render_viewport_data views[XRT_MAX_VIEWS]); @@ -1373,13 +1380,28 @@ render_compute_projection_timewarp(struct render_compute *render, * @public @memberof render_compute */ void -render_compute_projection(struct render_compute *render, - VkSampler src_samplers[XRT_MAX_VIEWS], - VkImageView src_image_views[XRT_MAX_VIEWS], - const struct xrt_normalized_rect src_rects[XRT_MAX_VIEWS], - VkImage target_image, - VkImageView target_image_view, - const struct render_viewport_data views[XRT_MAX_VIEWS]); +render_compute_projection_scanout_compensation(struct render_compute *render, + VkSampler src_samplers[XRT_MAX_VIEWS], + VkImageView src_image_views[XRT_MAX_VIEWS], + const struct xrt_normalized_rect src_rects[XRT_MAX_VIEWS], + const struct xrt_fov src_fovs[XRT_MAX_VIEWS], + const struct xrt_pose new_poses_scanout_begin[XRT_MAX_VIEWS], + const struct xrt_pose new_poses_scanout_end[XRT_MAX_VIEWS], + VkImage target_image, + VkImageView target_image_view, + const struct render_viewport_data views[XRT_MAX_VIEWS]); + +/*! + * @public @memberof render_compute + */ +void +render_compute_projection_no_timewarp(struct render_compute *render, + VkSampler src_samplers[XRT_MAX_VIEWS], + VkImageView src_image_views[XRT_MAX_VIEWS], + const struct xrt_normalized_rect src_rects[XRT_MAX_VIEWS], + VkImage target_image, + VkImageView target_image_view, + const struct render_viewport_data views[XRT_MAX_VIEWS]); /*! * @public @memberof render_compute diff --git a/src/xrt/compositor/render/render_util.c b/src/xrt/compositor/render/render_util.c index b567bf144..cb9804e49 100644 --- a/src/xrt/compositor/render/render_util.c +++ b/src/xrt/compositor/render/render_util.c @@ -123,6 +123,17 @@ render_calc_time_warp_matrix(const struct xrt_pose *src_pose, } } +void +render_calc_time_warp_projection(const struct xrt_fov *fov, struct xrt_matrix_4x4 *result) +{ + struct xrt_matrix_4x4_f64 tmp; + calc_projection(fov, &tmp); + + for (int i = 0; i < 16; i++) { + result->v[i] = (float)tmp.v[i]; + } +} + void render_calc_uv_to_tangent_lengths_rect(const struct xrt_fov *fov, struct xrt_normalized_rect *out_rect) { diff --git a/src/xrt/compositor/shaders/distortion.comp b/src/xrt/compositor/shaders/distortion.comp index 08b44d4cf..0b28d8bf3 100644 --- a/src/xrt/compositor/shaders/distortion.comp +++ b/src/xrt/compositor/shaders/distortion.comp @@ -24,8 +24,9 @@ layout(set = 0, binding = 3, std140) uniform restrict Config ivec4 views[2]; vec4 pre_transform[2]; vec4 post_transform[2]; - mat4 transform[2]; -} ubo; + mat4 transform_timewarp_scanout_begin[2]; + mat4 transform_timewarp_scanout_end[2]; +}ubo; vec2 position_to_uv(ivec2 extent, uint ix, uint iy) @@ -79,16 +80,14 @@ vec2 transform_uv_timewarp(vec2 uv, uint iz) values.xy = values.xy * ubo.pre_transform[iz].zw + ubo.pre_transform[iz].xy; values.y = -values.y; // Flip to OpenXR coordinate system. - // Timewarp. - values = ubo.transform[iz] * values; + // Timewarp including scanline timewarp for rolling refresh panels. + values = ubo.transform_timewarp_scanout_begin[iz] * values * (1 - uv.y) + + ubo.transform_timewarp_scanout_end[iz] * values * uv.y; values.xy = values.xy * (1.0 / max(values.w, 0.00001)); // From [-1, 1] to [0, 1] values.xy = values.xy * 0.5 + 0.5; - // To deal with OpenGL flip and sub image view. - values.xy = values.xy * ubo.post_transform[iz].zw + ubo.post_transform[iz].xy; - // Done. return values.xy; } @@ -96,10 +95,10 @@ vec2 transform_uv_timewarp(vec2 uv, uint iz) vec2 transform_uv(vec2 uv, uint iz) { if (do_timewarp) { - return transform_uv_timewarp(uv, iz); - } else { - return transform_uv_subimage(uv, iz); + uv = transform_uv_timewarp(uv, iz); } + + return transform_uv_subimage(uv, iz); } void main() @@ -127,11 +126,11 @@ void main() b_uv = transform_uv(b_uv, iz); // Sample the source with distorted and chromatic-aberration corrected samples. - vec4 colour = vec4( - texture(source[iz], r_uv).r, - texture(source[iz], g_uv).g, - texture(source[iz], b_uv).b, - 1); + vec4 colour = vec4( // + texture(source[iz], r_uv).r, // + texture(source[iz], g_uv).g, // + texture(source[iz], b_uv).b, // + 1); // // Do colour correction here since there are no automatic conversion in hardware available. colour = vec4(from_linear_to_srgb(colour.rgb), 1); diff --git a/src/xrt/compositor/util/comp_high_level_render.c b/src/xrt/compositor/util/comp_high_level_render.c index f1c9aa145..d8bf54569 100644 --- a/src/xrt/compositor/util/comp_high_level_render.c +++ b/src/xrt/compositor/util/comp_high_level_render.c @@ -147,7 +147,8 @@ chl_frame_state_gfx_set_target(struct chl_frame_state *frame_state, void chl_frame_state_cs_set_views(struct chl_frame_state *frame_state, - const struct xrt_pose world_poses[XRT_MAX_VIEWS], + const struct xrt_pose world_poses_scanout_begin[XRT_MAX_VIEWS], + const struct xrt_pose world_poses_scanout_end[XRT_MAX_VIEWS], const struct xrt_pose eye_poses[XRT_MAX_VIEWS], const struct xrt_fov fovs[XRT_MAX_VIEWS], uint32_t layer_count) @@ -172,14 +173,15 @@ chl_frame_state_cs_set_views(struct chl_frame_state *frame_state, VkImageView storage_view = comp_scratch_single_images_get_storage_view(scratch_view, scratch_index); - comp_render_cs_add_squash_view( // - &frame_state->data, // - &world_poses[i], // - &eye_poses[i], // - &fovs[i], // - rsci->image, // squash_image - storage_view, // squash_storage_view - &layer_viewport_data); // squash_viewport_data + comp_render_cs_add_squash_view( // + &frame_state->data, // + &world_poses_scanout_begin[i], // + &world_poses_scanout_end[i], // + &eye_poses[i], // + &fovs[i], // + rsci->image, // squash_image + storage_view, // squash_storage_view + &layer_viewport_data); // squash_viewport_data if (layer_count == 0) { frame_state->scratch_state.views[i].used = false; diff --git a/src/xrt/compositor/util/comp_high_level_render.h b/src/xrt/compositor/util/comp_high_level_render.h index 46edfd669..55ee67f78 100644 --- a/src/xrt/compositor/util/comp_high_level_render.h +++ b/src/xrt/compositor/util/comp_high_level_render.h @@ -172,7 +172,8 @@ chl_frame_state_gfx_default_pipeline(struct chl_frame_state *frame_state, */ void chl_frame_state_cs_set_views(struct chl_frame_state *frame_state, - const struct xrt_pose world_pose[XRT_MAX_VIEWS], + const struct xrt_pose world_pose_scanout_begin[XRT_MAX_VIEWS], + const struct xrt_pose world_pose_scanout_end[XRT_MAX_VIEWS], const struct xrt_pose eye_pose[XRT_MAX_VIEWS], const struct xrt_fov fov[XRT_MAX_VIEWS], uint32_t layer_count); @@ -200,19 +201,21 @@ chl_frame_state_cs_default_pipeline(struct chl_frame_state *frame_state, struct render_compute *render, const struct comp_layer *layers, uint32_t layer_count, - const struct xrt_pose world_poses[XRT_MAX_VIEWS], + const struct xrt_pose world_poses_scanout_begin[XRT_MAX_VIEWS], + const struct xrt_pose world_poses_scanout_end[XRT_MAX_VIEWS], const struct xrt_pose eye_poses[XRT_MAX_VIEWS], const struct xrt_fov fovs[XRT_MAX_VIEWS], VkImage target_image, VkImageView target_storage_view, const struct render_viewport_data target_viewport_datas[XRT_MAX_VIEWS]) { - chl_frame_state_cs_set_views( // - frame_state, // - world_poses, // - eye_poses, // - fovs, // - layer_count); // + chl_frame_state_cs_set_views( // + frame_state, // + world_poses_scanout_begin, // + world_poses_scanout_end, // + eye_poses, // + fovs, // + layer_count); // chl_frame_state_cs_set_target( // frame_state, // diff --git a/src/xrt/compositor/util/comp_render.h b/src/xrt/compositor/util/comp_render.h index 334464256..bd7ab2957 100644 --- a/src/xrt/compositor/util/comp_render.h +++ b/src/xrt/compositor/util/comp_render.h @@ -71,8 +71,11 @@ struct render_gfx_target_resources; */ struct comp_render_view_data { - //! New world pose of this view. - struct xrt_pose world_pose; + //! New world pose of this view at the beginng of scanout. + struct xrt_pose world_pose_scanout_begin; + + //! New world pose of this view at the end of scanout. + struct xrt_pose world_pose_scanout_end; //! New eye pose of this view. struct xrt_pose eye_pose; @@ -250,7 +253,8 @@ comp_render_initial_init(struct comp_render_dispatch_data *data, bool fast_path, */ static inline struct comp_render_view_data * comp_render_dispatch_add_squash_view(struct comp_render_dispatch_data *data, - const struct xrt_pose *world_pose, + const struct xrt_pose *world_pose_scanout_begin, + const struct xrt_pose *world_pose_scanout_end, const struct xrt_pose *eye_pose, const struct xrt_fov *fov, VkImage squash_image, @@ -265,7 +269,8 @@ comp_render_dispatch_add_squash_view(struct comp_render_dispatch_data *data, render_calc_uv_to_tangent_lengths_rect(fov, &view->pre_transform); // Common - view->world_pose = *world_pose; + view->world_pose_scanout_begin = *world_pose_scanout_begin; + view->world_pose_scanout_end = *world_pose_scanout_end; view->eye_pose = *eye_pose; view->fov = *fov; @@ -373,6 +378,7 @@ comp_render_gfx_add_squash_view(struct comp_render_dispatch_data *data, struct comp_render_view_data *view = comp_render_dispatch_add_squash_view( // data, // world_pose, // + world_pose, // eye_pose, // fov, // squash_image, // @@ -517,10 +523,12 @@ comp_render_cs_add_target(struct comp_render_dispatch_data *data, VkImage target * Add view to the common data, as required by the CS renderer. * * @param[in,out] data Common render dispatch data, will be updated - * @param world_pose New world pose of this view. + * @param world_pose_scanout_begin New world pose of this view. + * Populates @ref comp_render_view_data::world_pose + * @param world_pose_scanout_end New world pose of this view. * Populates @ref comp_render_view_data::world_pose * @param eye_pose New eye pose of this view - * Populates @ref comp_render_view_data::eye_pose + * Populates @ref comp_render_view_data::eye_pose_scanout_end * @param fov Assigned to fov in the view data, and used to compute @ref comp_render_view_data::pre_transform. * Populates @ref comp_render_view_data::fov * @param squash_image Scratch image for this view @@ -537,7 +545,8 @@ comp_render_cs_add_target(struct comp_render_dispatch_data *data, VkImage target */ static inline void comp_render_cs_add_squash_view(struct comp_render_dispatch_data *data, - const struct xrt_pose *world_pose, + const struct xrt_pose *world_pose_scanout_begin, + const struct xrt_pose *world_pose_scanout_end, const struct xrt_pose *eye_pose, const struct xrt_fov *fov, VkImage squash_image, @@ -546,7 +555,8 @@ comp_render_cs_add_squash_view(struct comp_render_dispatch_data *data, { struct comp_render_view_data *view = comp_render_dispatch_add_squash_view( // data, // - world_pose, // + world_pose_scanout_begin, // + world_pose_scanout_end, // eye_pose, // fov, // squash_image, // @@ -604,7 +614,8 @@ comp_render_cs_layer(struct render_compute *render, const struct comp_layer *layers, const uint32_t layer_count, const struct xrt_normalized_rect *pre_transform, - const struct xrt_pose *world_pose, + const struct xrt_pose *world_pose_scanout_begin, + const struct xrt_pose *world_pose_scanout_end, const struct xrt_pose *eye_pose, const VkImage target_image, const VkImageView target_image_view, diff --git a/src/xrt/compositor/util/comp_render_cs.c b/src/xrt/compositor/util/comp_render_cs.c index 4e20b32f0..f7da44e7f 100644 --- a/src/xrt/compositor/util/comp_render_cs.c +++ b/src/xrt/compositor/util/comp_render_cs.c @@ -185,7 +185,7 @@ do_cs_equirect2_layer(const struct comp_layer *layer, /// Data setup for a projection layer static inline void do_cs_projection_layer(const struct comp_layer *layer, - const struct xrt_pose *world_pose, + const struct xrt_pose *world_pose_scanout_begin, uint32_t view_index, uint32_t cur_layer, uint32_t cur_image, @@ -235,11 +235,11 @@ do_cs_projection_layer(const struct comp_layer *layer, // unused if timewarp is off if (do_timewarp) { - render_calc_time_warp_matrix( // - &vd->pose, // - &vd->fov, // - world_pose, // - &ubo_data->transforms[cur_layer]); // + render_calc_time_warp_matrix( // + &vd->pose, // + &vd->fov, // + world_pose_scanout_begin, // + &ubo_data->transforms_timewarp[cur_layer]); // } *out_cur_image = cur_image; @@ -373,6 +373,9 @@ crc_distortion_after_squash(struct render_compute *render, const struct comp_ren VkSampler src_samplers[XRT_MAX_VIEWS]; struct render_viewport_data target_viewport_datas[XRT_MAX_VIEWS]; struct xrt_normalized_rect src_norm_rects[XRT_MAX_VIEWS]; + struct xrt_fov src_fovs[XRT_MAX_VIEWS]; + struct xrt_pose world_poses_scanout_begin[XRT_MAX_VIEWS]; + struct xrt_pose world_poses_scanout_end[XRT_MAX_VIEWS]; for (uint32_t i = 0; i < d->target.view_count; i++) { // Data to be filled in. @@ -390,16 +393,36 @@ crc_distortion_after_squash(struct render_compute *render, const struct comp_ren src_norm_rects[i] = src_norm_rect; src_samplers[i] = clamp_to_border_black; target_viewport_datas[i] = viewport_data; + + if (d->do_timewarp) { + world_poses_scanout_begin[i] = d->views[i].world_pose_scanout_begin; + world_poses_scanout_end[i] = d->views[i].world_pose_scanout_end; + src_fovs[i] = d->views[i].fov; + } } - render_compute_projection( // - render, // - src_samplers, // - src_image_views, // - src_norm_rects, // - d->target.cs.image, // - d->target.cs.storage_view, // target_image_view - target_viewport_datas); // views + if (!d->do_timewarp) { + render_compute_projection_no_timewarp( // + render, // + src_samplers, // + src_image_views, // + src_norm_rects, // + d->target.cs.image, // + d->target.cs.storage_view, // target_image_view + target_viewport_datas); // views + } else { + render_compute_projection_scanout_compensation( // + render, // + src_samplers, // + src_image_views, // + src_norm_rects, // + src_fovs, // + world_poses_scanout_begin, // + world_poses_scanout_end, // + d->target.cs.image, // + d->target.cs.storage_view, // target_image_view + target_viewport_datas); // views + } } /// Fast path @@ -427,7 +450,8 @@ crc_distortion_fast_path(struct render_compute *render, struct xrt_normalized_rect src_norm_rects[XRT_MAX_VIEWS]; struct xrt_fov src_fovs[XRT_MAX_VIEWS]; struct xrt_pose src_poses[XRT_MAX_VIEWS]; - struct xrt_pose world_poses[XRT_MAX_VIEWS]; + struct xrt_pose world_poses_scanout_begin[XRT_MAX_VIEWS]; + struct xrt_pose world_poses_scanout_end[XRT_MAX_VIEWS]; for (uint32_t i = 0; i < d->target.view_count; i++) { // Data to be filled in. @@ -436,7 +460,8 @@ crc_distortion_fast_path(struct render_compute *render, struct xrt_normalized_rect src_norm_rect; struct xrt_fov src_fov; struct xrt_pose src_pose; - struct xrt_pose world_pose; + struct xrt_pose world_pose_scanout_begin; + struct xrt_pose world_pose_scanout_end; uint32_t array_index = vds[i]->sub.array_index; const struct comp_swapchain_image *image = get_layer_image(layer, i, vds[i]->sub.image_index); @@ -446,7 +471,8 @@ crc_distortion_fast_path(struct render_compute *render, viewport_data = d->views[i].target.viewport_data; src_fov = vds[i]->fov; src_pose = vds[i]->pose; - world_pose = d->views[i].world_pose; + world_pose_scanout_begin = d->views[i].world_pose_scanout_begin; + world_pose_scanout_end = d->views[i].world_pose_scanout_end; // No layer squasher has handled this for us already if (data->flip_y) { @@ -461,18 +487,19 @@ crc_distortion_fast_path(struct render_compute *render, target_viewport_datas[i] = viewport_data; src_fovs[i] = src_fov; src_poses[i] = src_pose; - world_poses[i] = world_pose; + world_poses_scanout_begin[i] = world_pose_scanout_begin; + world_poses_scanout_end[i] = world_pose_scanout_end; } if (!d->do_timewarp) { - render_compute_projection( // - render, // - src_samplers, // - src_image_views, // - src_norm_rects, // - d->target.cs.image, // - d->target.cs.storage_view, // - target_viewport_datas); // + render_compute_projection_no_timewarp( // + render, // + src_samplers, // + src_image_views, // + src_norm_rects, // + d->target.cs.image, // + d->target.cs.storage_view, // + target_viewport_datas); // } else { render_compute_projection_timewarp( // render, // @@ -481,7 +508,8 @@ crc_distortion_fast_path(struct render_compute *render, src_norm_rects, // src_poses, // src_fovs, // - world_poses, // + world_poses_scanout_begin, // + world_poses_scanout_end, // d->target.cs.image, // d->target.cs.storage_view, // target_viewport_datas); // @@ -501,7 +529,8 @@ comp_render_cs_layer(struct render_compute *render, const struct comp_layer *layers, const uint32_t layer_count, const struct xrt_normalized_rect *pre_transform, - const struct xrt_pose *world_pose, + const struct xrt_pose *world_pose_scanout_begin, + const struct xrt_pose *world_pose_scanout_end, const struct xrt_pose *eye_pose, const VkImage target_image, const VkImageView target_image_view, @@ -512,9 +541,9 @@ comp_render_cs_layer(struct render_compute *render, VkSampler clamp_to_border_black = render->r->samplers.clamp_to_border_black; // Not the transform of the views, but the inverse: actual view matrices. - struct xrt_matrix_4x4 world_view_mat, eye_view_mat; - math_matrix_4x4_view_from_pose(world_pose, &world_view_mat); - math_matrix_4x4_view_from_pose(eye_pose, &eye_view_mat); + struct xrt_matrix_4x4 world_view_mat_scanout_begin, eye_view; + math_matrix_4x4_view_from_pose(world_pose_scanout_begin, &world_view_mat_scanout_begin); + math_matrix_4x4_view_from_pose(eye_pose, &eye_view); struct render_buffer *ubo = &render->r->compute.layer.ubos[view_index]; struct render_compute_layer_ubo_data *ubo_data = ubo->mapped; @@ -564,65 +593,65 @@ comp_render_cs_layer(struct render_compute *render, switch (data->type) { case XRT_LAYER_CYLINDER: - do_cs_cylinder_layer( // - layer, // layer - &eye_view_mat, // eye_view_mat - &world_view_mat, // world_view_mat - view_index, // view_index - cur_layer, // cur_layer - cur_image, // cur_image - clamp_to_edge, // clamp_to_edge - clamp_to_border_black, // clamp_to_border_black - src_samplers, // src_samplers - src_image_views, // src_image_views - ubo_data, // ubo_data - &cur_image); // out_cur_image + do_cs_cylinder_layer( // + layer, // layer + &eye_view, // eye_view_mat + &world_view_mat_scanout_begin, // world_view_mat + view_index, // view_index + cur_layer, // cur_layer + cur_image, // cur_image + clamp_to_edge, // clamp_to_edge + clamp_to_border_black, // clamp_to_border_black + src_samplers, // src_samplers + src_image_views, // src_image_views + ubo_data, // ubo_data + &cur_image); // out_cur_image break; case XRT_LAYER_EQUIRECT2: - do_cs_equirect2_layer( // - layer, // layer - &eye_view_mat, // eye_view_mat - &world_view_mat, // world_view_mat - view_index, // view_index - cur_layer, // cur_layer - cur_image, // cur_image - clamp_to_edge, // clamp_to_edge - clamp_to_border_black, // clamp_to_border_black - src_samplers, // src_samplers - src_image_views, // src_image_views - ubo_data, // ubo_data - &cur_image); // out_cur_image + do_cs_equirect2_layer( // + layer, // layer + &eye_view, // eye_view_mat + &world_view_mat_scanout_begin, // world_view_mat + view_index, // view_index + cur_layer, // cur_layer + cur_image, // cur_image + clamp_to_edge, // clamp_to_edge + clamp_to_border_black, // clamp_to_border_black + src_samplers, // src_samplers + src_image_views, // src_image_views + ubo_data, // ubo_data + &cur_image); // out_cur_image break; case XRT_LAYER_PROJECTION_DEPTH: case XRT_LAYER_PROJECTION: { - do_cs_projection_layer( // - layer, // layer - world_pose, // world_pose - view_index, // view_index - cur_layer, // cur_layer - cur_image, // cur_image - clamp_to_edge, // clamp_to_edge - clamp_to_border_black, // clamp_to_border_black - src_samplers, // src_samplers - src_image_views, // src_image_views - ubo_data, // ubo_data - do_timewarp, // do_timewarp - &cur_image); // out_cur_image + do_cs_projection_layer( // + layer, // layer + world_pose_scanout_begin, // world_pose_scanout_begin + view_index, // view_index + cur_layer, // cur_layer + cur_image, // cur_image + clamp_to_edge, // clamp_to_edge + clamp_to_border_black, // clamp_to_border_black + src_samplers, // src_samplers + src_image_views, // src_image_views + ubo_data, // ubo_data + do_timewarp, // do_timewarp + &cur_image); // out_cur_image } break; case XRT_LAYER_QUAD: { - do_cs_quad_layer( // - layer, // layer - &eye_view_mat, // eye_view_mat - &world_view_mat, // world_view_mat - view_index, // view_index - cur_layer, // cur_layer - cur_image, // cur_image - clamp_to_edge, // clamp_to_edge - clamp_to_border_black, // clamp_to_border_black - src_samplers, // src_samplers - src_image_views, // src_image_views - ubo_data, // ubo_data - &cur_image); // out_cur_image + do_cs_quad_layer( // + layer, // layer + &eye_view, // eye_view_mat + &world_view_mat_scanout_begin, // world_view_mat_scanout_begin + view_index, // view_index + cur_layer, // cur_layer + cur_image, // cur_image + clamp_to_edge, // clamp_to_edge + clamp_to_border_black, // clamp_to_border_black + src_samplers, // src_samplers + src_image_views, // src_image_views + ubo_data, // ubo_data + &cur_image); // out_cur_image } break; default: // Should not get here! @@ -687,18 +716,19 @@ comp_render_cs_layers(struct render_compute *render, for (uint32_t view_index = 0; view_index < d->squash_view_count; view_index++) { const struct comp_render_view_data *view = &d->views[view_index]; - comp_render_cs_layer( // - render, // - view_index, // - layers, // - layer_count, // - &view->pre_transform, // - &view->world_pose, // - &view->eye_pose, // - view->squash.image, // - view->squash.cs.storage_view, // - &view->squash.viewport_data, // - d->do_timewarp); // + comp_render_cs_layer( // + render, // + view_index, // + layers, // + layer_count, // + &view->pre_transform, // + &view->world_pose_scanout_begin, // + &view->world_pose_scanout_end, // + &view->eye_pose, // + view->squash.image, // + view->squash.cs.storage_view, // + &view->squash.viewport_data, // + d->do_timewarp); // } cmd_barrier_view_squash_images( // diff --git a/src/xrt/compositor/util/comp_render_gfx.c b/src/xrt/compositor/util/comp_render_gfx.c index 63537c927..e0ab04248 100644 --- a/src/xrt/compositor/util/comp_render_gfx.c +++ b/src/xrt/compositor/util/comp_render_gfx.c @@ -531,11 +531,11 @@ crg_distortion_common(struct render_gfx *render, if (do_timewarp) { data.pre_transform = d->views[i].pre_transform; - render_calc_time_warp_matrix( // - &md->views[i].src_pose, // - &md->views[i].src_fov, // - &d->views[i].world_pose, // - &data.transform); // + render_calc_time_warp_matrix( // + &md->views[i].src_pose, // + &md->views[i].src_fov, // + &d->views[i].world_pose_scanout_begin, // + &data.transform); // } ret = render_gfx_mesh_alloc_and_write( // @@ -596,7 +596,7 @@ crg_distortion_after_squash(struct render_gfx *render, const struct comp_render_ struct gfx_mesh_data md = XRT_STRUCT_INIT; for (uint32_t i = 0; i < d->target.view_count; i++) { - struct xrt_pose src_pose = d->views[i].world_pose; + struct xrt_pose src_pose = d->views[i].world_pose_scanout_begin; struct xrt_fov src_fov = d->views[i].fov; VkImageView src_image_view = d->views[i].squash_as_src.sample_view; struct xrt_normalized_rect src_norm_rect = d->views[i].squash_as_src.norm_rect; @@ -693,7 +693,7 @@ comp_render_gfx_layers(struct render_gfx *render, for (uint32_t view = 0; view < d->squash_view_count; view++) { // Data for this view, convenience. - const struct xrt_pose world_pose = d->views[view].world_pose; + const struct xrt_pose world_pose = d->views[view].world_pose_scanout_begin; const struct xrt_pose eye_pose = d->views[view].eye_pose; const struct xrt_fov new_fov = d->views[view].fov; diff --git a/src/xrt/drivers/android/android_sensors.c b/src/xrt/drivers/android/android_sensors.c index b422d05a8..ec712bdd0 100644 --- a/src/xrt/drivers/android/android_sensors.c +++ b/src/xrt/drivers/android/android_sensors.c @@ -392,6 +392,8 @@ android_device_create(void) } d->base.hmd->screens[0].nominal_frame_interval_ns = time_s_to_ns(1.0f / metrics.refresh_rate); + d->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_NONE; + d->base.hmd->screens[0].scanout_time_ns = 0; const uint32_t w_pixels = metrics.width_pixels; const uint32_t h_pixels = metrics.height_pixels; diff --git a/src/xrt/drivers/blubur_s1/blubur_s1_hmd.c b/src/xrt/drivers/blubur_s1/blubur_s1_hmd.c index 0db776545..7eb45eae1 100644 --- a/src/xrt/drivers/blubur_s1/blubur_s1_hmd.c +++ b/src/xrt/drivers/blubur_s1/blubur_s1_hmd.c @@ -543,6 +543,8 @@ blubur_s1_hmd_create(struct os_hid_device *dev, const char *serial) hmd->base.hmd->screens[0].w_pixels = PANEL_WIDTH; hmd->base.hmd->screens[0].h_pixels = VIEW_SIZE; hmd->base.hmd->screens[0].nominal_frame_interval_ns = 1000000000LLU / 120; // 120hz + hmd->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_NONE; + hmd->base.hmd->screens[0].scanout_time_ns = 0; hmd->base.hmd->view_count = 2; hmd->base.hmd->views[0] = (struct xrt_view){ diff --git a/src/xrt/drivers/hdk/hdk_device.cpp b/src/xrt/drivers/hdk/hdk_device.cpp index a407259bb..7676f678d 100644 --- a/src/xrt/drivers/hdk/hdk_device.cpp +++ b/src/xrt/drivers/hdk/hdk_device.cpp @@ -355,6 +355,9 @@ hdk_device_create(struct os_hid_device *dev, enum HDK_VARIANT variant) hd->base.hmd->distortion.fov[0].angle_right = -hd->base.hmd->distortion.fov[1].angle_left; } + hd->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_NONE; + hd->base.hmd->screens[0].scanout_time_ns = 0; + switch (variant) { case HDK_UNKNOWN: assert(!"unknown device"); break; diff --git a/src/xrt/drivers/ohmd/oh_device.c b/src/xrt/drivers/ohmd/oh_device.c index 0aca05274..af0fd04e9 100644 --- a/src/xrt/drivers/ohmd/oh_device.c +++ b/src/xrt/drivers/ohmd/oh_device.c @@ -800,6 +800,8 @@ create_hmd(ohmd_context *ctx, int device_idx, int device_flags) ohd->base.hmd->screens[0].w_pixels = info.display.w_pixels; ohd->base.hmd->screens[0].h_pixels = info.display.h_pixels; ohd->base.hmd->screens[0].nominal_frame_interval_ns = info.display.nominal_frame_interval_ns; + ohd->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_NONE; + ohd->base.hmd->screens[0].scanout_time_ns = 0; // Left ohd->base.hmd->views[0].display.w_pixels = info.views[0].display.w_pixels; diff --git a/src/xrt/drivers/rift/rift_hmd.c b/src/xrt/drivers/rift/rift_hmd.c index 06bd4b59e..a0c7d4629 100644 --- a/src/xrt/drivers/rift/rift_hmd.c +++ b/src/xrt/drivers/rift/rift_hmd.c @@ -624,6 +624,8 @@ rift_hmd_create(struct os_hid_device *dev, enum rift_variant variant, char *devi // Set up display details hmd->base.hmd->screens[0].nominal_frame_interval_ns = time_s_to_ns(1.0f / 75.0f); + hmd->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_NONE; + hmd->base.hmd->screens[0].scanout_time_ns = 0; hmd->extra_display_info.icd = MICROMETERS_TO_METERS(hmd->display_info.lens_separation); diff --git a/src/xrt/drivers/rokid/rokid_hmd.c b/src/xrt/drivers/rokid/rokid_hmd.c index 905f38a11..60dfc2000 100644 --- a/src/xrt/drivers/rokid/rokid_hmd.c +++ b/src/xrt/drivers/rokid/rokid_hmd.c @@ -496,6 +496,8 @@ rokid_hmd_create(struct xrt_prober_device *prober_device) // Set up display details // refresh rate rokid->base.hmd->screens[0].nominal_frame_interval_ns = time_s_to_ns(1.0f / 60.0f); + rokid->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_NONE; + rokid->base.hmd->screens[0].scanout_time_ns = 0; const float quarter_vFOV = 0.25f * (is_rokid_max ? 46.0f : 40.0f) * ((float)M_PI / 180.0f); const float quarter_hFOV = quarter_vFOV * 16.0f / 9.0f; diff --git a/src/xrt/drivers/sample/sample_hmd.c b/src/xrt/drivers/sample/sample_hmd.c index 744573efc..ee2d26592 100644 --- a/src/xrt/drivers/sample/sample_hmd.c +++ b/src/xrt/drivers/sample/sample_hmd.c @@ -207,6 +207,8 @@ sample_hmd_create(void) // Set up display details // refresh rate hmd->base.hmd->screens[0].nominal_frame_interval_ns = time_s_to_ns(1.0f / 90.0f); + hmd->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_NONE; + hmd->base.hmd->screens[0].scanout_time_ns = 0; const double hFOV = 90 * (M_PI / 180.0); const double vFOV = 96.73 * (M_PI / 180.0); diff --git a/src/xrt/drivers/simula/svr_hmd.c b/src/xrt/drivers/simula/svr_hmd.c index 1c40c269c..8db47e8bf 100644 --- a/src/xrt/drivers/simula/svr_hmd.c +++ b/src/xrt/drivers/simula/svr_hmd.c @@ -245,6 +245,8 @@ svr_hmd_create(struct svr_two_displays_distortion *distortion) svr->base.device_type = XRT_DEVICE_TYPE_HMD; svr->base.hmd->screens[0].nominal_frame_interval_ns = (uint64_t)time_s_to_ns(1.0f / 90.0f); + svr->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_NONE; + svr->base.hmd->screens[0].scanout_time_ns = 0; // Print name. diff --git a/src/xrt/drivers/steamvr_lh/device.cpp b/src/xrt/drivers/steamvr_lh/device.cpp index a232381b9..3d3e36296 100644 --- a/src/xrt/drivers/steamvr_lh/device.cpp +++ b/src/xrt/drivers/steamvr_lh/device.cpp @@ -867,6 +867,26 @@ HmdDevice::set_nominal_frame_interval(uint64_t interval_ns) } } +void +HmdDevice::set_scanout_type(xrt_scanout_direction direction, uint64_t time_ns) +{ + auto set = [this, direction, time_ns] { + hmd_parts->base.screens[0].scanout_direction = direction; + hmd_parts->base.screens[0].scanout_time_ns = time_ns; + }; + + if (hmd_parts) { + set(); + } else { + std::thread t([this, set] { + std::unique_lock lk(hmd_parts_mut); + hmd_parts_cv.wait(lk, [this] { return hmd_parts != nullptr; }); + set(); + }); + t.detach(); + } +} + namespace { // From openvr driver documentation // (https://github.com/ValveSoftware/openvr/blob/master/docs/Driver_API_Documentation.md#Input-Profiles): @@ -1027,7 +1047,13 @@ HmdDevice::handle_property_write(const vr::PropertyWrite_t &prop) case vr::Prop_DisplayFrequency_Float: { assert(prop.unBufferSize == sizeof(float)); float freq = *static_cast(prop.pvBuffer); - set_nominal_frame_interval((1.f / freq) * 1e9f); + uint64_t interval_ns = (1.f / freq) * 1e9f; + set_nominal_frame_interval(interval_ns); + if (variant == VIVE_VARIANT_PRO) { + set_scanout_type(XRT_SCANOUT_DIRECTION_TOP_TO_BOTTOM, interval_ns * 1600.0 / 1624.0); + } else { + set_scanout_type(XRT_SCANOUT_DIRECTION_NONE, 0); + } break; } case vr::Prop_UserIpdMeters_Float: { diff --git a/src/xrt/drivers/steamvr_lh/device.hpp b/src/xrt/drivers/steamvr_lh/device.hpp index dab788570..75f3a21d5 100644 --- a/src/xrt/drivers/steamvr_lh/device.hpp +++ b/src/xrt/drivers/steamvr_lh/device.hpp @@ -210,6 +210,9 @@ private: void set_nominal_frame_interval(uint64_t interval_ns); + void + set_scanout_type(enum xrt_scanout_direction direction, uint64_t time_ns); + std::condition_variable hmd_parts_cv; std::mutex hmd_parts_mut; float brightness{1.0f}; diff --git a/src/xrt/drivers/survive/survive_driver.c b/src/xrt/drivers/survive/survive_driver.c index 4f5455d9c..fdd1c5f8e 100644 --- a/src/xrt/drivers/survive/survive_driver.c +++ b/src/xrt/drivers/survive/survive_driver.c @@ -979,6 +979,16 @@ _create_hmd_device(struct survive_system *sys, const struct SurviveSimpleObject survive->base.hmd->screens[0].nominal_frame_interval_ns = (uint64_t)time_s_to_ns(1.0f / 90.0f); } + if (survive->hmd.config.variant == VIVE_VARIANT_PRO) { + survive->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_TOP_TO_BOTTOM; + survive->base.hmd->screens[0].scanout_time_ns = survive->base.hmd->screens[0].nominal_frame_interval_ns; + // Compensate for the length of vblank. + survive->base.hmd->screens[0].scanout_time_ns *= 1600.0 / 1624.0; + } else { + survive->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_NONE; + survive->base.hmd->screens[0].scanout_time_ns = 0; + } + for (uint8_t eye = 0; eye < 2; eye++) { struct xrt_view *v = &survive->base.hmd->views[eye]; v->display.w_pixels = w_pixels; diff --git a/src/xrt/drivers/vive/vive_device.c b/src/xrt/drivers/vive/vive_device.c index 0b4974e22..18cc02f7d 100644 --- a/src/xrt/drivers/vive/vive_device.c +++ b/src/xrt/drivers/vive/vive_device.c @@ -1157,6 +1157,16 @@ vive_device_create(struct os_hid_device *mainboard_dev, d->base.hmd->screens[0].nominal_frame_interval_ns = (uint64_t)time_s_to_ns(1.0f / 90.0f); } + if (d->config.variant == VIVE_VARIANT_PRO) { + d->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_TOP_TO_BOTTOM; + d->base.hmd->screens[0].scanout_time_ns = d->base.hmd->screens[0].nominal_frame_interval_ns; + // Compensate for the length of vblank. + d->base.hmd->screens[0].scanout_time_ns *= 1600.0 / 1624.0; + } else { + d->base.hmd->screens[0].scanout_direction = XRT_SCANOUT_DIRECTION_NONE; + d->base.hmd->screens[0].scanout_time_ns = 0; + } + for (uint8_t eye = 0; eye < 2; eye++) { struct xrt_view *v = &d->base.hmd->views[eye]; v->display.w_pixels = w_pixels; diff --git a/src/xrt/drivers/xreal_air/xreal_air_hmd.c b/src/xrt/drivers/xreal_air/xreal_air_hmd.c index 47fd1afb0..7818c8406 100644 --- a/src/xrt/drivers/xreal_air/xreal_air_hmd.c +++ b/src/xrt/drivers/xreal_air/xreal_air_hmd.c @@ -1198,6 +1198,7 @@ xreal_air_hmd_create_device(struct os_hid_device *sensor_device, // Set up display details refresh rate hmd->base.hmd->screens[0].nominal_frame_interval_ns = time_s_to_ns(1.0f / 60.0f); + hmd->base.hmd->screens[0].scanout_time_ns = 0; // Distortion information, fills in xdev->compute_distortion(). u_distortion_mesh_set_none(&hmd->base); diff --git a/src/xrt/include/xrt/xrt_defines.h b/src/xrt/include/xrt/xrt_defines.h index 143ba5332..0f8ff8963 100644 --- a/src/xrt/include/xrt/xrt_defines.h +++ b/src/xrt/include/xrt/xrt_defines.h @@ -171,6 +171,20 @@ enum xrt_distortion_model // clang-format on }; +/*! + * Screen scanout direction + */ +enum xrt_scanout_direction +{ + // clang-format off + XRT_SCANOUT_DIRECTION_NONE = 0, + XRT_SCANOUT_DIRECTION_TOP_TO_BOTTOM, + XRT_SCANOUT_DIRECTION_BOTTOM_TO_TOP, + XRT_SCANOUT_DIRECTION_LEFT_TO_RIGHT, + XRT_SCANOUT_DIRECTION_RIGHT_TO_LEFT, + // clang-format on +}; + /*! * Common formats, use `u_format_*` functions to reason about them. */ diff --git a/src/xrt/include/xrt/xrt_device.h b/src/xrt/include/xrt/xrt_device.h index 15a46207a..47d9ce487 100644 --- a/src/xrt/include/xrt/xrt_device.h +++ b/src/xrt/include/xrt/xrt_device.h @@ -102,6 +102,8 @@ struct xrt_hmd_parts int h_pixels; //! Nominal frame interval uint64_t nominal_frame_interval_ns; + enum xrt_scanout_direction scanout_direction; + uint64_t scanout_time_ns; } screens[1]; /*! diff --git a/src/xrt/targets/sdl_test/sdl_device.c b/src/xrt/targets/sdl_test/sdl_device.c index b1d94fe4c..00a0ca9be 100644 --- a/src/xrt/targets/sdl_test/sdl_device.c +++ b/src/xrt/targets/sdl_test/sdl_device.c @@ -115,6 +115,7 @@ sdl_device_init(struct sdl_program *sp) // Refresh rate. xdev->hmd->screens[0].nominal_frame_interval_ns = time_s_to_ns(1.0f / 60.0f); + xdev->hmd->screens[0].scanout_time_ns = 0; // Blend mode(s), setup after u_device_setup_split_side_by_side. xdev->hmd->blend_modes[0] = XRT_BLEND_MODE_OPAQUE; -- 2.51.2