Something went wrong. Try again.
The open source OpenXR runtime
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286// Copyright 2022-2026, Collabora, Ltd.// SPDX-License-Identifier: BSL-1.0/*! * @file * @brief Mercury hand tracking main file. * @author Jakob Bornecrantz <jakob@collabora.com> * @author Moshi Turner <moshiturner@protonmail.com> * @author Charlton Rodda <charlton.rodda@collabora.com> * @ingroup tracking */
#include "hg_sync.hpp"#include "hg_image_math.inl"#include "tracking/t_hand_tracking.h"#include "util/u_box_iou.hpp"#include "util/u_hand_tracking.h"#include "math/m_vec2.h"#include "util/u_misc.h"#include "xrt/xrt_defines.h"#include "xrt/xrt_frame.h"#include "xrt/xrt_tracking.h"
#include <algorithm>#include <array>#include <numeric>
namespace xrt::tracking::hand::mercury {
DEBUG_GET_ONCE_LOG_OPTION(mercury_log, "MERCURY_LOG", U_LOGGING_WARN)DEBUG_GET_ONCE_BOOL_OPTION(mercury_optimize_hand_size, "MERCURY_optimize_hand_size", true)DEBUG_GET_ONCE_FLOAT_OPTION(mercury_min_detection_confidence, "MERCURY_MIN_DETECTION_CONFIDENCE", 0.3)
// Flags to tell state tracker that these are indeed valid jointsstatic const enum xrt_space_relation_flags valid_flags_ht = (enum xrt_space_relation_flags)( XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT | XRT_SPACE_RELATION_POSITION_VALID_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT);
/*! * Setup helper functions. */
static boolgetCalibration(struct HandTracking *hgt, t_stereo_camera_calibration &calibration){ xrt_vec3 trans = { (float)calibration.camera_translation[0], (float)calibration.camera_translation[1], (float)calibration.camera_translation[2], };
if (hgt->log_level <= U_LOGGING_DEBUG) { HG_DEBUG(hgt, "Dumping full camera calibration!"); t_stereo_camera_calibration_dump(&calibration); }
hgt->baseline = m_vec3_len(trans); HG_DEBUG(hgt, "I think the baseline is %f meters!", hgt->baseline);
{ // Officially, I have no idea if this is row-major or col-major. Empirically it seems to work, and that // is all I will say.
// It might be that the below is *transposing* the matrix, I never remember if OpenCV's R is "left in // right" or "right in left" //!@todo xrt_matrix_3x3 s;
s.v[0] = (float)calibration.camera_rotation[0][0]; s.v[1] = (float)calibration.camera_rotation[1][0]; s.v[2] = (float)calibration.camera_rotation[2][0];
s.v[3] = (float)calibration.camera_rotation[0][1]; s.v[4] = (float)calibration.camera_rotation[1][1]; s.v[5] = (float)calibration.camera_rotation[2][1];
s.v[6] = (float)calibration.camera_rotation[0][2]; s.v[7] = (float)calibration.camera_rotation[1][2]; s.v[8] = (float)calibration.camera_rotation[2][2];
xrt_pose left_in_right; left_in_right.position = trans;
math_quat_from_matrix_3x3(&s, &left_in_right.orientation);
//! @todo what are these magic values? //! they're probably turning the OpenCV formalism into OpenXR, but especially what gives with negating //! orientation.x? left_in_right.orientation.x = -left_in_right.orientation.x; left_in_right.position.y = -left_in_right.position.y; left_in_right.position.z = -left_in_right.position.z;
hgt->left_in_right = left_in_right;
HG_DEBUG(hgt, "left_in_right pose: %f %f %f %f %f %f %f", // left_in_right.position.x, left_in_right.position.y, left_in_right.position.z, // left_in_right.orientation.x, left_in_right.orientation.y, left_in_right.orientation.z, left_in_right.orientation.w); }
for (int view_idx = 0; view_idx < 2; view_idx++) { ht_view &view = hgt->views[view_idx]; t_camera_model_params_from_t_camera_calibration(&calibration.view[view_idx], &view.hgdist_orig);
view.hgdist = view.hgdist_orig; }
//!@todo Really? We can totally support cameras with varying resolutions. // For a later MR. hgt->calibration_one_view_size_px.w = calibration.view[0].image_size_pixels.w; hgt->calibration_one_view_size_px.h = calibration.view[0].image_size_pixels.h;
hgt->last_frame_one_view_size_px = hgt->calibration_one_view_size_px; hgt->multiply_px_coord_for_undistort = 1.0f;
hgt->hand_pose_camera_offset = XRT_POSE_IDENTITY; return true;}
static inline boolcheck_outside_view(struct HandTracking *hgt, struct t_camera_extra_info_one_view boundary, xrt_vec2 &keypoint){ // Regular case - the keypoint is literally outside the image if (keypoint.y > hgt->last_frame_one_view_size_px.h || // keypoint.y < 0 || // keypoint.x > hgt->last_frame_one_view_size_px.w || // keypoint.x < 0) { return true; }
switch (boundary.boundary_type) { // No boundary, and we passed the previous check. Not outside the view. case HT_IMAGE_BOUNDARY_NONE: return false; break; case HT_IMAGE_BOUNDARY_CIRCLE: { //!@todo Optimize: Most of this can be calculated once at startup xrt_vec2 center_px = { boundary.boundary.circle.normalized_center.x * (float)hgt->last_frame_one_view_size_px.w, // boundary.boundary.circle.normalized_center.y * (float)hgt->last_frame_one_view_size_px.h}; float radius_px = boundary.boundary.circle.normalized_radius * (float)hgt->last_frame_one_view_size_px.w;
xrt_vec2 keypoint_xrt = {float(keypoint.x), float(keypoint.y)};
xrt_vec2 diff = center_px - keypoint_xrt; if (m_vec2_len(diff) > radius_px) { return true; } } break; }
return false;}
/*! * Transform 3d joint keypoint positions into per-view camera coordinates * * @param[in,out] hgt Hand tracking global structure * @param[in] pts Keypoints (in world space relative to left camera?) * @param[in] hand_idx 0 is left, 1 is right * @param[in] also_debug_output if true, will also draw on the debug image * @param[out] num_outside optional count of keypoints outside of bounds per view */static voidback_project(struct HandTracking *hgt, // const Eigen::Array<float, 3, 21> &pts, // int hand_idx, // bool also_debug_output, // int num_outside[2]){
for (int view_idx = 0; view_idx < 2; view_idx++) { cv::Mat debug = hgt->views[view_idx].debug_out_to_this; xrt_pose move_amount = {};
if (view_idx == 0) { // left camera. move_amount = XRT_POSE_IDENTITY; } else { move_amount = hgt->left_in_right; }
Eigen::Vector3f p = map_vec3(move_amount.position); Eigen::Quaternionf q = map_quat(move_amount.orientation);
// 3 rows (xyz), 21 cols (on for each keypoint) Eigen::Array<float, 3, 21> pts_relative_to_camera = {};
bool invalid[21] = {};
// TODO is this sufficient? // Eigen::Matrix3f rot = q.matrix(); // pts_relative_to_camera = (pts.colwise() * rot).colwise() + p;
for (int i = 0; i < 21; i++) { pts_relative_to_camera.col(i) = (q * pts.col(i)) + p;
if (pts_relative_to_camera.col(i).z() > 0) { invalid[i] = true; } }
// in 2D camera space... normalized? // TODO these stay uninitialized if the point is already considered "invalid" - probable UB later xrt_vec2 keypoints_global[21];
for (int i = 0; i < 21; i++) { invalid[i] = invalid[i] || !t_camera_models_flip_and_project(&hgt->views[view_idx].hgdist, // pts_relative_to_camera.col(i).x(), // pts_relative_to_camera.col(i).y(), // pts_relative_to_camera.col(i).z(), // &keypoints_global[i].x, // &keypoints_global[i].y // ); }
for (int i = 0; i < 21; i++) { invalid[i] = invalid[i] || check_outside_view(hgt, hgt->views[view_idx].camera_info, keypoints_global[i]); }
if (num_outside != NULL) { num_outside[view_idx] = std::count(std::begin(invalid), std::end(invalid), true);
xrt_vec2 min = keypoints_global[0]; xrt_vec2 max = keypoints_global[0];
for (int i = 0; i < 21; i++) { xrt_vec2 &pt = keypoints_global[i]; min.x = fmin(pt.x, min.x); min.y = fmin(pt.y, min.y);
max.x = fmax(pt.x, max.x); max.y = fmax(pt.y, max.y); } xrt_vec2 center = m_vec2_mul_scalar(min + max, 0.5);
float r = fmax(center.x - min.x, center.y - min.y);
float size = r * 2;
hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].center_px = center; hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].size_px = size; if (also_debug_output) { handSquare(debug, center, size, GREEN); } }
if (also_debug_output) { for (int i = 0; i < 21; i++) { xrt_vec2 loc; loc.x = keypoints_global[i].x; loc.y = keypoints_global[i].y; handDot(debug, loc, 2, invalid[i] ? 0.0 : (float)(i) / 26.0, 1, 2); } } } // for view_idx}
static voidback_project_keypoint_output(struct HandTracking *hgt, // int hand_idx, // int view_idx){
cv::Mat debug = hgt->views[view_idx].debug_out_to_this; one_frame_one_view &view = hgt->keypoint_outputs[hand_idx].views[view_idx];
for (int i = 0; i < 21; i++) {
//!@todo We're trivially rewriting the stereographic projection for like the 2nd or 3rd time here. We //! should add an Eigen template for this instead.
xrt_vec2 dir_sg = m_vec2_mul_scalar(view.keypoints_in_scaled_stereographic[i].pos_2d, view.stereographic_radius);
float denom = (1 + dir_sg.x * dir_sg.x + dir_sg.y * dir_sg.y); xrt_vec3 dir = {}; dir.x = 2 * dir_sg.x / denom; dir.y = 2 * dir_sg.y / denom; dir.z = (-1 + (dir_sg.x * dir_sg.x) + (dir_sg.y * dir_sg.y)) / denom;
math_quat_rotate_vec3(&view.look_dir, &dir, &dir);
xrt_vec2 loc = {}; t_camera_models_flip_and_project(&hgt->views[view_idx].hgdist, // dir.x, // dir.y, // dir.z, // &loc.x, // &loc.y // );
handDot(debug, loc, 2, (float)(i) / 26.0, 1, 2); }}
static boolhandle_changed_image_size(HandTracking *hgt, xrt_size &new_one_view_size){ int gcd_calib = std::gcd(hgt->calibration_one_view_size_px.h, hgt->calibration_one_view_size_px.w); int gcd_new = std::gcd(new_one_view_size.h, new_one_view_size.w);
int lcm_h_calib = hgt->calibration_one_view_size_px.h / gcd_calib; int lcm_w_calib = hgt->calibration_one_view_size_px.w / gcd_calib;
int lcm_h_new = new_one_view_size.h / gcd_new; int lcm_w_new = new_one_view_size.w / gcd_new;
bool good = (lcm_h_calib == lcm_h_new) && (lcm_w_calib == lcm_w_new);
if (!good) { HG_WARN(hgt, "Can't process this frame, wrong aspect ratio. What we wanted: %dx%d, what we got: %dx%d", lcm_h_calib, lcm_w_calib, lcm_h_new, lcm_w_new); return false; }
//!@todo optimize: can't we just scale camera matrix/etc correctly? hgt->multiply_px_coord_for_undistort = (float)hgt->calibration_one_view_size_px.h / (float)new_one_view_size.h; hgt->last_frame_one_view_size_px = new_one_view_size;
for (int view_idx = 0; view_idx < 2; view_idx++) { hgt->views[view_idx].hgdist.fx = hgt->views[view_idx].hgdist_orig.fx / hgt->multiply_px_coord_for_undistort; hgt->views[view_idx].hgdist.fy = hgt->views[view_idx].hgdist_orig.fy / hgt->multiply_px_coord_for_undistort;
hgt->views[view_idx].hgdist.cx = hgt->views[view_idx].hgdist_orig.cx / hgt->multiply_px_coord_for_undistort; hgt->views[view_idx].hgdist.cy = hgt->views[view_idx].hgdist_orig.cy / hgt->multiply_px_coord_for_undistort; } return true;}
xrt_vec3correct_direction(xrt_vec2 in){ xrt_vec3 out = {in.x, -in.y, -1}; return m_vec3_normalize(out);}
voidcheck_new_user_event(struct HandTracking *hgt){ if (hgt->tuneable_values.new_user_event) { hgt->tuneable_values.new_user_event = false; hgt->hand_seen_before[0] = false; hgt->hand_seen_before[1] = false; hgt->hand_size_refinement.newUserEvent(); }}
/// The area the intersection of these regions of interest, relative to the combined area (Intersection Over Union)static floathand_bounding_boxes_iou(const hand_region_of_interest &one, const hand_region_of_interest &two){ if (!one.found || !two.found) { return -1; } box_iou::Box this_box(one.center_px, one.size_px); box_iou::Box other_box(two.center_px, two.size_px);
return boxIOU(this_box, other_box);}
voiddispatch_and_process_hand_detections(struct HandTracking *hgt){ if (hgt->tuneable_values.always_run_detection_model) { // Pretend like nothing was detected last frame. for (int hand_idx = 0; hand_idx < 2; hand_idx++) { hgt->this_frame_hand_detected[hand_idx] = false;
hgt->history_hands[hand_idx].clear(); } }
hand_detection_run_info infos[2] = {};
// Mega paranoia, should get optimized out. for (int view_idx = 0; view_idx < 2; view_idx++) { for (int hand_idx = 0; hand_idx < 2; hand_idx++) { infos[view_idx].outputs[hand_idx].found = false; infos[view_idx].outputs[hand_idx].hand_detection_confidence = 0; infos[view_idx].outputs[hand_idx].provenance = ROIProvenance::HAND_DETECTION; } }
infos[0].view = &hgt->views[0]; infos[1].view = &hgt->views[1];
bool no_hands_detected_last_frame = !(hgt->this_frame_hand_detected[0] || hgt->this_frame_hand_detected[1]);
size_t active_camera = hgt->detection_counter++ % 2;
int num_views = 0;
if (hgt->tuneable_values.always_run_detection_model || hgt->hand_size_refinement.isOptimizing() || hgt->tuneable_values.detection_model_in_both_views) { u_worker_group_push(hgt->group, run_hand_detection, &infos[0]); u_worker_group_push(hgt->group, run_hand_detection, &infos[1]); num_views = 2; u_worker_group_wait_all(hgt->group); } else { run_hand_detection(&infos[active_camera]); num_views = 1; }
for (int hand_idx = 0; hand_idx < 2; hand_idx++) { float confidence_sum = (infos[0].outputs[hand_idx].hand_detection_confidence + infos[1].outputs[hand_idx].hand_detection_confidence) / float(num_views); if (confidence_sum < 0.92) { continue; }
if (hgt->tuneable_values.always_run_detection_model || !hgt->last_frame_hand_detected[hand_idx]) {
bool good_to_go = true;
if (no_hands_detected_last_frame) { // Stop overlapping _double_ hand detections - detecting both hands in the same place. // This happens a lot if you put your hands together (we can't track intertwining hands // yet) for (int view_idx = 0; view_idx < 2; view_idx++) { float iou = hand_bounding_boxes_iou(infos[view_idx].outputs[hand_idx], infos[view_idx].outputs[!hand_idx]); if (iou > hgt->tuneable_values.mpiou_double_detection.val) { HG_DEBUG(hgt, "Rejected double detection because the iou for hand idx %d, " "view idx " "%d was %f", hand_idx, view_idx, iou); good_to_go = false; break; } } } else { // Stop overlapping _single_ hand detections - detecting one hand where another hand is // already tracked. This happens a lot if you trick the hand detector into thinking your // left hand is a right hand. for (int view_idx = 0; view_idx < 2; view_idx++) { hand_region_of_interest &this_state = infos[view_idx].outputs[hand_idx];
// Note that this is not just the other state hand_region_of_interest &other_state = hgt->views[view_idx].regions_of_interest_this_frame[!hand_idx]; float iou = hand_bounding_boxes_iou(this_state, other_state); if (iou > hgt->tuneable_values.mpiou_single_detection.val) { HG_DEBUG(hgt, "Rejected single detection because the iou for hand idx %d, " "view idx %d was %f", hand_idx, view_idx, iou); good_to_go = false; break; } } }
if (good_to_go) { // Note we already initialized the previous-keypoints-input to nonexistent above this. hgt->views[0].regions_of_interest_this_frame[hand_idx] = infos[0].outputs[hand_idx]; hgt->views[1].regions_of_interest_this_frame[hand_idx] = infos[1].outputs[hand_idx]; } }
hgt->this_frame_hand_detected[hand_idx] = true; }}
voidhand_joint_set_to_eigen_21(const xrt_hand_joint_set &set, Eigen::Array<float, 3, 21> &out){ int acc_idx = 0;
out.col(acc_idx++) = map_vec3(set.values.hand_joint_set_default[XRT_HAND_JOINT_WRIST].relation.pose.position); for (int finger = 0; finger < 5; finger++) { for (int joint = 1; joint < 5; joint++) { xrt_hand_joint j = joints_5x5_to_26[finger][joint]; const xrt_vec3 &jp = set.values.hand_joint_set_default[j].relation.pose.position; out.col(acc_idx++) = map_vec3(jp); } }}
// Most of the time, this codepath runs - we predict where the hand should be based on the last// two frames.voidpredict_new_regions_of_interest(struct HandTracking *hgt){
xrt_hand_masks_sample masks{}; // Zero initialization
for (int hand_idx = 0; hand_idx < 2; hand_idx++) { // If we don't have the past two frames, this code doesn't do what we want. // If we only have *one* frame, we just reuse the same bounding box and hope the hand // hasn't moved too much. @todo
auto &hh = hgt->history_hands[hand_idx];
if (hh.size() < 2) { HG_TRACE(hgt, "continuing, size is %zu", hgt->history_hands[hand_idx].size()); continue; }
// We can only do this *after* we know we're predicting - this would otherwise overwrite the detection // model. hgt->this_frame_hand_detected[hand_idx] = hgt->last_frame_hand_detected[hand_idx];
uint64_t time_two_frames_ago = *hgt->history_timestamps.get_at_age(1); uint64_t time_one_frame_ago = *hgt->history_timestamps.get_at_age(0); uint64_t time_now = hgt->current_frame_timestamp;
// double dt_past = (double)() / (double)U_TIME_1S_IN_NS; double dt_past = time_ns_to_s(time_one_frame_ago - time_two_frames_ago);
double dt_now = time_ns_to_s(time_now - time_one_frame_ago);
Eigen::Array<float, 3, 21> &n_minus_two = *hh.get_at_age(1); Eigen::Array<float, 3, 21> &n_minus_one = *hh.get_at_age(0);
Eigen::Array<float, 3, 21> add;
add = n_minus_one - n_minus_two;
add *= (dt_now * hgt->tuneable_values.amount_to_lerp_prediction.val) / dt_past;
hgt->pose_predicted_keypoints[hand_idx] = n_minus_one + add;
int num_outside[2]; back_project(hgt, hgt->pose_predicted_keypoints[hand_idx], hand_idx, hgt->tuneable_values.scribble_predictions_into_next_frame && hgt->debug_scribble, num_outside);
for (int view_idx = 0; view_idx < 2; view_idx++) { hand_region_of_interest &hroi = hgt->views[view_idx].regions_of_interest_this_frame[hand_idx]; auto &masks_view = masks.views[view_idx]; auto &masks_hand = masks_view.hands[hand_idx];
masks_view.enabled = true;
if (num_outside[view_idx] < hgt->tuneable_values.max_num_outside_view) { hroi.provenance = ROIProvenance::POSE_PREDICTION; hroi.found = true;
// Scale factor for uncertainty const float SCALER = 1.25f; float s = hroi.size_px * SCALER; xrt_vec2 &c = hroi.center_px; masks_hand.rect = xrt_rect_f32{c.x - s / 2, c.y - s / 2, s, s}; masks_hand.enabled = true; } else { hroi.found = false; masks_hand.enabled = false; } } }
if (hgt->hand_masks_sink != NULL) { xrt_sink_push_hand_masks(hgt->hand_masks_sink, &masks); }}
//!@todo This looks like it sucks, but it doesn't given the current architecture.// There are two distinct failure modes here:// * One hand goes over the other hand, and we wish to discard the hand that is being obscured.// * One hand "ate" the other hand: easiest way to see this is by putting your hands close together and shaking them// around.//// If we were only concerned about the first one, we'd do some simple depth testing to figure out which one is// closer to the hand and only discard the further-away hand. But the second one is such a common (and bad)// failure mode that we really just need to stop tracking all hands if they start overlapping.
//!@todo I really want to try making a discrete optimizer that looks at recent info and decides whether to drop tracking//! for a hand, switch its handedness or switch to some forthcoming overlapping-hands model. This would likely work by//! pruning impossible combinations, calculating a loss for each remaining option and picking the least bad one.voidstop_everything_if_hands_are_overlapping(struct HandTracking *hgt){ bool ok = true; for (int view_idx = 0; view_idx < 2; view_idx++) { hand_region_of_interest left_box = hgt->views[view_idx].regions_of_interest_this_frame[0]; hand_region_of_interest right_box = hgt->views[view_idx].regions_of_interest_this_frame[1]; if (!left_box.found || !right_box.found) { continue; } box_iou::Box this_nbox(left_box.center_px, left_box.size_px); box_iou::Box other_nbox(right_box.center_px, right_box.size_px); float iou = box_iou::boxIOU(this_nbox, other_nbox); if (iou > hgt->tuneable_values.mpiou_any.val) { HG_DEBUG(hgt, "Stopped tracking because iou was %f in view %d", iou, view_idx); ok = false; break; } } if (!ok) { for (int view_idx = 0; view_idx < 2; view_idx++) { for (int hand_idx = 0; hand_idx < 2; hand_idx++) { hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].found = false; } } }}
boolhand_too_far(struct HandTracking *hgt, xrt_hand_joint_set &set){ xrt_vec3 dp = set.values.hand_joint_set_default[XRT_HAND_JOINT_PALM].relation.pose.position; return (m_vec3_len(dp) > hgt->tuneable_values.max_hand_dist.val);}
voidscribble_image_boundary(struct HandTracking *hgt){ for (int view_idx = 0; view_idx < 2; view_idx++) { struct ht_view *view = &hgt->views[view_idx];
cv::Mat &debug_frame = view->debug_out_to_this; t_camera_extra_info_one_view &info = hgt->views[view_idx].camera_info;
if (info.boundary_type == HT_IMAGE_BOUNDARY_CIRCLE) { int center_x = hgt->last_frame_one_view_size_px.w * info.boundary.circle.normalized_center.x; int center_y = hgt->last_frame_one_view_size_px.h * info.boundary.circle.normalized_center.y; cv::circle(debug_frame, {center_x, center_y}, info.boundary.circle.normalized_radius * hgt->last_frame_one_view_size_px.w, hsv2rgb(270.0, 0.5, 0.5), 2); } }}
/* * * Member functions. * */
HandTracking::HandTracking(){ this->base.process = &HandTracking::cCallbackProcess; this->base.destroy = &HandTracking::cCallbackDestroy; u_sink_debug_init(&this->debug_sink_ann); u_sink_debug_init(&this->debug_sink_model);}
HandTracking::~HandTracking(){ u_sink_debug_destroy(&this->debug_sink_ann); u_sink_debug_destroy(&this->debug_sink_model);
xrt_frame_reference(&this->visualizers.old_frame, NULL);
// Stop all the workers before we release the ONNX state u_worker_group_reference(&this->group, NULL);
release_onnx_state(&this->views[0].keypoint[0]); release_onnx_state(&this->views[0].keypoint[1]); release_onnx_state(&this->views[0].detection);
release_onnx_state(&this->views[1].keypoint[0]); release_onnx_state(&this->views[1].keypoint[1]); release_onnx_state(&this->views[1].detection);
t_stereo_camera_calibration_reference(&this->calib, NULL);
lm::optimizer_destroy(&this->kinematic_hands[0]); lm::optimizer_destroy(&this->kinematic_hands[1]);
u_var_remove_root((void *)&this->base); u_frame_times_widget_teardown(&this->ft_widget);}
static voidcallback_process_unsafe(HandTracking *hgt, struct xrt_frame *left_frame, struct xrt_frame *right_frame, struct xrt_hand_joint_set *out_left_hand, struct xrt_hand_joint_set *out_right_hand, int64_t *out_timestamp_ns){
struct xrt_hand_joint_set *out_xrt_hands[2] = {out_left_hand, out_right_hand};
/* * Setup views. */
assert(left_frame->width == right_frame->width); assert(left_frame->height == right_frame->height);
const int full_height = left_frame->height; const int full_width = left_frame->width * 2;
if ((left_frame->width != (uint32_t)hgt->last_frame_one_view_size_px.w) || (left_frame->height != (uint32_t)hgt->last_frame_one_view_size_px.h)) { xrt_size new_one_view_size; new_one_view_size.h = left_frame->height; new_one_view_size.w = left_frame->width; // Could be an assert, should never happen after first frame. if (!handle_changed_image_size(hgt, new_one_view_size)) { return; } }
const int view_width = hgt->last_frame_one_view_size_px.w; const int view_height = hgt->last_frame_one_view_size_px.h;
const cv::Size full_size = cv::Size(full_width, full_height); const cv::Size view_size = cv::Size(view_width, view_height); const cv::Point view_offsets[2] = {cv::Point(0, 0), cv::Point(view_width, 0)};
hgt->views[0].run_model_on_this = cv::Mat(view_size, CV_8UC1, left_frame->data, left_frame->stride); hgt->views[1].run_model_on_this = cv::Mat(view_size, CV_8UC1, right_frame->data, right_frame->stride);
hgt->debug_scribble = u_sink_debug_is_active(&hgt->debug_sink_ann) && u_sink_debug_is_active(&hgt->debug_sink_model);
cv::Mat debug_output = {}; xrt_frame *debug_frame = nullptr;
// If we're outputting to a debug image, setup the image. if (hgt->debug_scribble) { u_frame_create_one_off(XRT_FORMAT_R8G8B8, full_width, full_height, &debug_frame); debug_frame->timestamp = hgt->current_frame_timestamp;
debug_output = cv::Mat(full_size, CV_8UC3, debug_frame->data, debug_frame->stride);
cv::cvtColor(hgt->views[0].run_model_on_this, debug_output(cv::Rect(view_offsets[0], view_size)), cv::COLOR_GRAY2BGR); cv::cvtColor(hgt->views[1].run_model_on_this, debug_output(cv::Rect(view_offsets[1], view_size)), cv::COLOR_GRAY2BGR);
hgt->views[0].debug_out_to_this = debug_output(cv::Rect(view_offsets[0], view_size)); hgt->views[1].debug_out_to_this = debug_output(cv::Rect(view_offsets[1], view_size)); scribble_image_boundary(hgt);
// Let's check that the collage size is actually as big as we think it is static_assert(1064 == (8 + ((128 + 8) * 4) + ((320 + 8)) + ((80 + 8) * 2) + 8)); static_assert(504 == (240 + 240 + 8 + 8 + 8)); static_assert(552 == (8 + (128 + 8) * 4));
const int w = 1064; const int h = 552;
u_frame_create_one_off(XRT_FORMAT_L8, w, h, &hgt->visualizers.xrtframe); hgt->visualizers.xrtframe->timestamp = hgt->current_frame_timestamp;
cv::Size size = cv::Size(w, h);
hgt->visualizers.mat = cv::Mat(size, CV_8U, hgt->visualizers.xrtframe->data, hgt->visualizers.xrtframe->stride);
if (hgt->visualizers.old_frame == NULL) { // There wasn't a previous frame so let's setup the background hgt->visualizers.mat = 255; } else { // They had better be the same size. memcpy(hgt->visualizers.xrtframe->data, hgt->visualizers.old_frame->data, hgt->visualizers.old_frame->size); xrt_frame_reference(&hgt->visualizers.old_frame, NULL); } }
check_new_user_event(hgt);
// Every now and then if we're not already tracking both hands, try to detect new hands. bool saw_both_hands_last_frame = hgt->last_frame_hand_detected[0] && hgt->last_frame_hand_detected[1]; if (!saw_both_hands_last_frame || hgt->tuneable_values.always_run_detection_model) { dispatch_and_process_hand_detections(hgt); }
stop_everything_if_hands_are_overlapping(hgt);
//!@todo does this go here? // If no hand regions of interest were found anywhere, there's no hand - register that in the state // tracker for (int hand_idx = 0; hand_idx < 2; hand_idx++) { if (!(hgt->views[0].regions_of_interest_this_frame[hand_idx].found || hgt->views[1].regions_of_interest_this_frame[hand_idx].found)) { hgt->this_frame_hand_detected[hand_idx] = false; } }
// Dispatch keypoint estimator neural nets for (int hand_idx = 0; hand_idx < 2; hand_idx++) { for (int view_idx = 0; view_idx < 2; view_idx++) { if (!hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].found) { continue; }
struct keypoint_estimation_run_info &inf = hgt->views[view_idx].run_info[hand_idx]; inf.view = &hgt->views[view_idx]; inf.hand_idx = hand_idx; u_worker_group_push(hgt->group, hgt->keypoint_estimation_run_func, &hgt->views[view_idx].run_info[hand_idx]); } } u_worker_group_wait_all(hgt->group);
// Hand size refinement/optimization state bool optimize_hand_size; { constexpr size_t kLH = 0; constexpr size_t kRH = 1;
// input data
const std::array<HandSizeRefinement::ViewData, kNumViews> view_data{ HandSizeRefinement::ViewData{{ {hgt->views[0].regions_of_interest_this_frame[kLH].found}, {hgt->views[0].regions_of_interest_this_frame[kRH].found}, }}, HandSizeRefinement::ViewData{{ {hgt->views[1].regions_of_interest_this_frame[kLH].found}, {hgt->views[1].regions_of_interest_this_frame[kRH].found}, }}, };
const std::array<HandSizeRefinement::HandData, kNumHands> hand_data{ HandSizeRefinement::HandData{ .this_frame_hand_detected = hgt->this_frame_hand_detected[0], .hand_seen_before = hgt->hand_seen_before[0], }, HandSizeRefinement::HandData{ .this_frame_hand_detected = hgt->this_frame_hand_detected[1], .hand_seen_before = hgt->hand_seen_before[1], }, };
optimize_hand_size = hgt->hand_size_refinement.framePreOptimizer(view_data, hand_data);
hgt->target_hand_size = hgt->hand_size_refinement.getTargetHandSize();
optimize_hand_size = optimize_hand_size && hgt->tuneable_values.optimize_hand_size; }
// Dispatch the optimizers! for (int hand_idx = 0; hand_idx < 2; hand_idx++) {
for (int view_idx = 0; view_idx < 2; view_idx++) { if (!hgt->views[view_idx].regions_of_interest_this_frame[hand_idx].found) { // to the next view continue; }
if (!hgt->keypoint_outputs[hand_idx].views[view_idx].active) { HG_DEBUG(hgt, "Removing hand %d because keypoint estimator said to!", hand_idx); hgt->this_frame_hand_detected[hand_idx] = false; } }
if (!hgt->this_frame_hand_detected[hand_idx]) { continue; }
// propagate "no ROI" to "no keypoint outputs" // TODO seems redundant? for (int view = 0; view < 2; view++) { hand_region_of_interest &from_model = hgt->views[view].regions_of_interest_this_frame[hand_idx]; if (!from_model.found) { hgt->keypoint_outputs[hand_idx].views[view].active = false; } }
if (hgt->tuneable_values.scribble_keypoint_model_outputs && hgt->debug_scribble) { for (int view_idx = 0; view_idx < 2; view_idx++) {
if (!hgt->keypoint_outputs[hand_idx].views[view_idx].active) { continue; }
back_project_keypoint_output(hgt, hand_idx, view_idx); } }
struct xrt_hand_joint_set *put_in_set = out_xrt_hands[hand_idx];
lm::KinematicHandLM *hand = hgt->kinematic_hands[hand_idx];
float reprojection_error_threshold = hgt->tuneable_values.max_reprojection_error.val; float smoothing_factor = hgt->tuneable_values.opt_smooth_factor.val;
if (hgt->last_frame_hand_detected[hand_idx]) { if (hgt->tuneable_values.enable_framerate_based_smoothing) { int64_t one_before = *hgt->history_timestamps.get_at_age(0); int64_t now = hgt->current_frame_timestamp;
uint64_t diff = now - one_before; double diff_d = time_ns_to_s(diff); smoothing_factor = hgt->tuneable_values.opt_smooth_factor.val * (1 / 60.0f) / diff_d; } }
float out_hand_size;
//!@todo optimize: We can have one of these on each thread float reprojection_error; lm::optimizer_run(hand, // hgt->keypoint_outputs[hand_idx], // !hgt->last_frame_hand_detected[hand_idx], // smoothing_factor, optimize_hand_size, // hgt->target_hand_size, // hgt->hand_size_refinement.getHandSizeErrorMul(), // hgt->tuneable_values.amt_use_depth.val, *put_in_set, // out_hand_size, // reprojection_error);
if (reprojection_error > reprojection_error_threshold) { HG_DEBUG(hgt, "Reprojection error above threshold!"); hgt->this_frame_hand_detected[hand_idx] = false; continue; }
if (hand_too_far(hgt, *put_in_set)) { HG_DEBUG(hgt, "Hand too far away"); hgt->this_frame_hand_detected[hand_idx] = false; continue; }
hgt->hand_size_refinement.frameHandPostOptimizer(out_hand_size, reprojection_error, hgt->keypoint_outputs[hand_idx]);
u_hand_joints_apply_joint_width(put_in_set);
put_in_set->hand_pose.pose = hgt->hand_pose_camera_offset; put_in_set->hand_pose.relation_flags = valid_flags_ht;
Eigen::Array<float, 3, 21> asf = {};
hand_joint_set_to_eigen_21(*put_in_set, asf);
back_project(hgt, // asf, // hand_idx, // hgt->tuneable_values.scribble_optimizer_outputs && hgt->debug_scribble, // NULL // );
hgt->history_hands[hand_idx].push_back(asf); hgt->hand_tracked_for_num_frames[hand_idx]++; }
// Push our timestamp back as well hgt->history_timestamps.push_back(hgt->current_frame_timestamp);
hgt->hand_size_refinement.framePost();
// State tracker tweaks for (int hand_idx = 0; hand_idx < 2; hand_idx++) { out_xrt_hands[hand_idx]->is_active = hgt->this_frame_hand_detected[hand_idx]; hgt->last_frame_hand_detected[hand_idx] = hgt->this_frame_hand_detected[hand_idx];
hgt->hand_seen_before[hand_idx] = hgt->hand_seen_before[hand_idx] || hgt->this_frame_hand_detected[hand_idx];
if (!hgt->last_frame_hand_detected[hand_idx]) { hgt->views[0].regions_of_interest_this_frame[hand_idx].found = false; hgt->views[1].regions_of_interest_this_frame[hand_idx].found = false; hgt->history_hands[hand_idx].clear(); hgt->hand_tracked_for_num_frames[hand_idx] = 0; } }
// estimators next frame. Also, if next frame's hand will be outside of the camera's field of view, mark it as // inactive this frame. This stops issues where our hand detector detects hands that are slightly too close to // the edge, causing flickery hands. if (!hgt->tuneable_values.always_run_detection_model) { predict_new_regions_of_interest(hgt); bool still_found[2] = {hgt->last_frame_hand_detected[0], hgt->last_frame_hand_detected[1]}; still_found[0] = hgt->views[0].regions_of_interest_this_frame[0].found || hgt->views[1].regions_of_interest_this_frame[0].found; still_found[1] = hgt->views[0].regions_of_interest_this_frame[1].found || hgt->views[1].regions_of_interest_this_frame[1].found;
for (int hand_idx = 0; hand_idx < 2; hand_idx++) { out_xrt_hands[hand_idx]->is_active = still_found[hand_idx]; } }
for (int hand_idx = 0; hand_idx < 2; hand_idx++) { // Don't send the hand to OpenXR until it's been tracked for 4 frames if (hgt->hand_tracked_for_num_frames[hand_idx] < hgt->tuneable_values.num_frames_before_display) { out_xrt_hands[hand_idx]->is_active = false; } }
// If the debug UI is active, push to the frame-timing widget u_frame_times_widget_push_sample(&hgt->ft_widget, hgt->current_frame_timestamp);
// If the debug UI is active, push our debug frame if (hgt->debug_scribble) { u_sink_debug_push_frame(&hgt->debug_sink_ann, debug_frame); xrt_frame_reference(&debug_frame, NULL);
// We don't dereference the model inputs/outputs frame here; we make a copy of it next frame and // dereference it then. u_sink_debug_push_frame(&hgt->debug_sink_model, hgt->visualizers.xrtframe); xrt_frame_reference(&hgt->visualizers.old_frame, hgt->visualizers.xrtframe); xrt_frame_reference(&hgt->visualizers.xrtframe, NULL); }
// done!}
voidHandTracking::cCallbackProcess(struct t_hand_tracking_sync *ht_sync, struct xrt_frame *left_frame, struct xrt_frame *right_frame, struct xrt_hand_joint_set *out_left_hand, struct xrt_hand_joint_set *out_right_hand, int64_t *out_timestamp_ns){ XRT_TRACE_MARKER();
HandTracking *hgt = &HandTracking::fromC(ht_sync);
hgt->current_frame_timestamp = left_frame->timestamp;
// No filtering, fine to do this now. Also just a reminder that this took you 2 HOURS TO DEBUG THAT ONE TIME. *out_timestamp_ns = hgt->current_frame_timestamp;
try { callback_process_unsafe(hgt, left_frame, right_frame, out_left_hand, out_right_hand, out_timestamp_ns); } catch (const std::exception &e) { HG_ERROR(hgt, "Exception in hand tracking process callback: %s", e.what());
// Mark the hands as not active since an exception occurred. out_left_hand->is_active = false; out_right_hand->is_active = false; }}
voidHandTracking::cCallbackDestroy(t_hand_tracking_sync *ht_sync){ HandTracking *ht_ptr = &HandTracking::fromC(ht_sync);
try { delete ht_ptr; } catch (const std::exception &e) { HG_ERROR(ht_ptr, "Exception in hand tracking destroy callback: %s", e.what()); }}
} // namespace xrt::tracking::hand::mercury
using namespace xrt::tracking::hand::mercury;
/* * * 'Exported' functions. * */
extern "C" t_hand_tracking_sync *t_hand_tracking_sync_mercury_create(struct t_stereo_camera_calibration *calib, struct t_hand_tracking_create_info create_info, const char *models_folder){ XRT_TRACE_MARKER();
xrt::tracking::hand::mercury::HandTracking *hgt = new xrt::tracking::hand::mercury::HandTracking();
// Setup logging first. We like logging. hgt->log_level = xrt::tracking::hand::mercury::debug_get_log_option_mercury_log();
/* * Get configuration */
assert(calib != NULL); hgt->calib = NULL; // We have to reference it, getCalibration points at it. t_stereo_camera_calibration_reference(&hgt->calib, calib); getCalibration(hgt, *calib); strncpy(hgt->models_folder, models_folder, ARRAY_SIZE(hgt->models_folder) - 1);
hgt->views[0].hgt = hgt; hgt->views[1].hgt = hgt; // :)
hgt->hand_masks_sink = create_info.masks_sink;
struct t_camera_extra_info &extra_camera_info = create_info.cams_info; hgt->views[0].camera_info = extra_camera_info.views[0]; hgt->views[1].camera_info = extra_camera_info.views[1];
try { init_hand_detection(hgt, &hgt->views[0].detection); init_hand_detection(hgt, &hgt->views[1].detection);
init_keypoint_estimation(hgt, &hgt->views[0].keypoint[0]); init_keypoint_estimation(hgt, &hgt->views[0].keypoint[1]);
init_keypoint_estimation(hgt, &hgt->views[1].keypoint[0]); init_keypoint_estimation(hgt, &hgt->views[1].keypoint[1]); } catch (const std::exception &e) { HG_ERROR(hgt, "Failed to initialize ONNX runtime: %s", e.what()); delete hgt; return NULL; } hgt->keypoint_estimation_run_func = xrt::tracking::hand::mercury::run_keypoint_estimation;
hgt->views[0].view = 0; hgt->views[1].view = 1;
int num_threads = 4; hgt->pool = u_worker_thread_pool_create(num_threads - 1, num_threads, "Hand Tracking"); hgt->group = u_worker_group_create(hgt->pool);
lm::optimizer_create(hgt->left_in_right, false, hgt->log_level, &hgt->kinematic_hands[0]); lm::optimizer_create(hgt->left_in_right, true, hgt->log_level, &hgt->kinematic_hands[1]);
u_frame_times_widget_init(&hgt->ft_widget, 10.0f, 10.0f);
u_var_add_root(hgt, "Camera-based Hand Tracker", true);
u_var_add_ro_f32(hgt, &hgt->ft_widget.fps, "FPS!"); u_var_add_f32_timing(hgt, hgt->ft_widget.debug_var, "Frame timing!");
u_var_add_ro_f32(hgt, &hgt->target_hand_size, "Hand size (Meters between wrist and middle-proximal joint)");
float *hand_size_refinement_schedule_x = nullptr; float *hand_size_refinement_schedule_y = nullptr; hgt->hand_size_refinement.get_refinement_schedule_ptrs(hand_size_refinement_schedule_x, hand_size_refinement_schedule_y);
u_var_add_ro_f32(hgt, hand_size_refinement_schedule_x, "Schedule (X value)"); u_var_add_ro_f32(hgt, hand_size_refinement_schedule_y, "Schedule (Y value)");
u_var_add_bool(hgt, &hgt->tuneable_values.new_user_event, "Estimate hand sizes");
hgt->tuneable_values.optimize_hand_size = debug_get_bool_option_mercury_optimize_hand_size();
hgt->tuneable_values.dyn_radii_fac.max = 4.0f; hgt->tuneable_values.dyn_radii_fac.min = 0.3f; hgt->tuneable_values.dyn_radii_fac.step = 0.02f; hgt->tuneable_values.dyn_radii_fac.val = 1.7f;
hgt->tuneable_values.after_detection_fac.max = 1.0f; hgt->tuneable_values.after_detection_fac.min = 0.1f; hgt->tuneable_values.after_detection_fac.step = 0.01f; // note that sqrt2/2 is what should make sense, but I tuned it down to this. Detection model needs work. hgt->tuneable_values.after_detection_fac.val = 0.65f; // but 0.5 is closer to what we actually want
hgt->tuneable_values.dyn_joint_y_angle_error.max = 40.0f; hgt->tuneable_values.dyn_joint_y_angle_error.min = 0.0f; hgt->tuneable_values.dyn_joint_y_angle_error.step = 0.1f; hgt->tuneable_values.dyn_joint_y_angle_error.val = 10.0f;
// Number of times this has been changed without rigorously testing: 1 hgt->tuneable_values.amount_to_lerp_prediction.max = 1.5f; hgt->tuneable_values.amount_to_lerp_prediction.min = -1.5f; hgt->tuneable_values.amount_to_lerp_prediction.step = 0.01f; hgt->tuneable_values.amount_to_lerp_prediction.val = 0.4f;
hgt->tuneable_values.amt_use_depth.max = 1.0f; hgt->tuneable_values.amt_use_depth.min = 0.0f; hgt->tuneable_values.amt_use_depth.step = 0.01f; hgt->tuneable_values.amt_use_depth.val = 0.01f;
hgt->tuneable_values.mpiou_any.max = 1.0f; hgt->tuneable_values.mpiou_any.min = 0.0f; hgt->tuneable_values.mpiou_any.step = 0.01f; hgt->tuneable_values.mpiou_any.val = 0.7f;
hgt->tuneable_values.mpiou_double_detection.max = 1.0f; hgt->tuneable_values.mpiou_double_detection.min = 0.0f; hgt->tuneable_values.mpiou_double_detection.step = 0.01f; hgt->tuneable_values.mpiou_double_detection.val = 0.4f;
hgt->tuneable_values.mpiou_single_detection.max = 1.0f; hgt->tuneable_values.mpiou_single_detection.min = 0.0f; hgt->tuneable_values.mpiou_single_detection.step = 0.01f; hgt->tuneable_values.mpiou_single_detection.val = 0.2f;
hgt->tuneable_values.max_reprojection_error.max = 600.0f; hgt->tuneable_values.max_reprojection_error.min = 0.0f; hgt->tuneable_values.max_reprojection_error.step = 0.001f; hgt->tuneable_values.max_reprojection_error.val = 15.0f;
hgt->tuneable_values.opt_smooth_factor.max = 30.0f; hgt->tuneable_values.opt_smooth_factor.min = 0.0f; hgt->tuneable_values.opt_smooth_factor.step = 0.01f; hgt->tuneable_values.opt_smooth_factor.val = 2.0f;
hgt->tuneable_values.max_hand_dist.max = 1000000.0f; hgt->tuneable_values.max_hand_dist.min = 0.0f; hgt->tuneable_values.max_hand_dist.step = 0.05f; hgt->tuneable_values.max_hand_dist.val = 1.7f;
hgt->tuneable_values.min_detection_confidence.max = 1.0f; hgt->tuneable_values.min_detection_confidence.min = 0.0f; hgt->tuneable_values.min_detection_confidence.step = 0.01f; hgt->tuneable_values.min_detection_confidence.val = debug_get_float_option_mercury_min_detection_confidence();
u_var_add_draggable_f32(hgt, &hgt->tuneable_values.amt_use_depth, "Amount to use depth prediction");
u_var_add_draggable_f32(hgt, &hgt->tuneable_values.dyn_radii_fac, "radius factor (predict)"); u_var_add_draggable_f32(hgt, &hgt->tuneable_values.after_detection_fac, "radius factor (after hand detection)"); u_var_add_draggable_f32(hgt, &hgt->tuneable_values.dyn_joint_y_angle_error, "max error hand joint"); u_var_add_draggable_f32(hgt, &hgt->tuneable_values.amount_to_lerp_prediction, "Amount to lerp pose-prediction"); u_var_add_draggable_f32(hgt, &hgt->tuneable_values.mpiou_any, "Max permissible IOU (Any)"); u_var_add_draggable_f32(hgt, &hgt->tuneable_values.mpiou_double_detection, "Max permissible IOU (For suppressing double detections)"); u_var_add_draggable_f32(hgt, &hgt->tuneable_values.mpiou_single_detection, "Max permissible IOU (For suppressing single detections)"); u_var_add_draggable_f32(hgt, &hgt->tuneable_values.max_reprojection_error, "Max reprojection error"); u_var_add_draggable_f32(hgt, &hgt->tuneable_values.opt_smooth_factor, "Optimizer smoothing factor"); u_var_add_draggable_f32(hgt, &hgt->tuneable_values.max_hand_dist, "Max hand distance"); u_var_add_draggable_f32(hgt, &hgt->tuneable_values.min_detection_confidence, "Min detection confidence");
u_var_add_i32(hgt, &hgt->tuneable_values.max_num_outside_view, "max allowed number of hand joints outside view"); u_var_add_u64(hgt, &hgt->tuneable_values.num_frames_before_display, "Number of frames before we show hands to OpenXR");
u_var_add_bool(hgt, &hgt->tuneable_values.scribble_predictions_into_next_frame, "Scribble pose-predictions into next frame"); u_var_add_bool(hgt, &hgt->tuneable_values.scribble_keypoint_model_outputs, "Scribble keypoint model output"); u_var_add_bool(hgt, &hgt->tuneable_values.scribble_optimizer_outputs, "Scribble kinematic optimizer output"); u_var_add_bool(hgt, &hgt->tuneable_values.always_run_detection_model, "Use detection model instead of pose-predicting into next frame"); u_var_add_bool(hgt, &hgt->tuneable_values.optimize_hand_size, "Optimize hand size"); u_var_add_bool(hgt, &hgt->tuneable_values.enable_pose_predicted_input, "Enable pose-predicted input to keypoint model"); u_var_add_bool(hgt, &hgt->tuneable_values.enable_framerate_based_smoothing, "Enable framerate-based smoothing (Don't use; surprisingly seems to make things worse)"); u_var_add_bool(hgt, &hgt->tuneable_values.detection_model_in_both_views, "Run detection model in both views ");
u_var_add_sink_debug(hgt, &hgt->debug_sink_ann, "Annotated camera feeds"); u_var_add_sink_debug(hgt, &hgt->debug_sink_model, "Model inputs and outputs");
HG_DEBUG(hgt, "Hand Tracker initialized!");
return &hgt->base;}