From aec911a9425b35b212da3575de17ebddd37f19b4 Mon Sep 17 00:00:00 2001 From: Korcan Hussein Date: Fri, 24 Oct 2025 17:17:33 +0100 Subject: [PATCH] xrt: implements xrt/ipc plumbing for XR_ANDROID_face_tracking Part-of: --- src/xrt/include/xrt/xrt_defines.h | 103 ++++++++++++++++++ src/xrt/include/xrt/xrt_device.h | 23 ++++ src/xrt/ipc/server/ipc_server_handler.c | 11 ++ src/xrt/ipc/shared/proto/50-device.json | 9 ++ .../oxr/oxr_face_tracking_android.c | 84 ++++++++++++++ src/xrt/state_trackers/oxr/oxr_objects.h | 3 + src/xrt/state_trackers/oxr/oxr_system.c | 36 ++++++ 7 files changed, 269 insertions(+) diff --git a/src/xrt/include/xrt/xrt_defines.h b/src/xrt/include/xrt/xrt_defines.h index e9eaa6f00..439e0ba53 100644 --- a/src/xrt/include/xrt/xrt_defines.h +++ b/src/xrt/include/xrt/xrt_defines.h @@ -1181,6 +1181,7 @@ enum xrt_input_type _(XRT_INPUT_HTC_LIP_FACE_TRACKING , XRT_INPUT_NAME(0x0602, FACE_TRACKING)) \ _(XRT_INPUT_FB_FACE_TRACKING2_AUDIO , XRT_INPUT_NAME(0x0603, FACE_TRACKING)) \ _(XRT_INPUT_FB_FACE_TRACKING2_VISUAL , XRT_INPUT_NAME(0x0604, FACE_TRACKING)) \ + _(XRT_INPUT_ANDROID_FACE_TRACKING , XRT_INPUT_NAME(0x0605, FACE_TRACKING)) \ \ _(XRT_INPUT_GENERIC_BODY_TRACKING , XRT_INPUT_NAME(0x0700, BODY_TRACKING)) \ _(XRT_INPUT_FB_BODY_TRACKING , XRT_INPUT_NAME(0x0701, BODY_TRACKING)) \ @@ -1764,6 +1765,96 @@ enum xrt_facial_tracking_type_htc #define XRT_FACIAL_EXPRESSION_EYE_COUNT_HTC 14 #define XRT_FACIAL_EXPRESSION_LIP_COUNT_HTC 37 +enum xrt_face_confidence_regions_android +{ + XRT_FACE_CONFIDENCE_REGIONS_LOWER_ANDROID = 0, + XRT_FACE_CONFIDENCE_REGIONS_LEFT_UPPER_ANDROID = 1, + XRT_FACE_CONFIDENCE_REGIONS_RIGHT_UPPER_ANDROID = 2, +}; + +enum xrt_face_parameter_indices_android +{ + XRT_FACE_PARAMETER_INDICES_BROW_LOWERER_L_ANDROID = 0, + XRT_FACE_PARAMETER_INDICES_BROW_LOWERER_R_ANDROID = 1, + XRT_FACE_PARAMETER_INDICES_CHEEK_PUFF_L_ANDROID = 2, + XRT_FACE_PARAMETER_INDICES_CHEEK_PUFF_R_ANDROID = 3, + XRT_FACE_PARAMETER_INDICES_CHEEK_RAISER_L_ANDROID = 4, + XRT_FACE_PARAMETER_INDICES_CHEEK_RAISER_R_ANDROID = 5, + XRT_FACE_PARAMETER_INDICES_CHEEK_SUCK_L_ANDROID = 6, + XRT_FACE_PARAMETER_INDICES_CHEEK_SUCK_R_ANDROID = 7, + XRT_FACE_PARAMETER_INDICES_CHIN_RAISER_B_ANDROID = 8, + XRT_FACE_PARAMETER_INDICES_CHIN_RAISER_T_ANDROID = 9, + XRT_FACE_PARAMETER_INDICES_DIMPLER_L_ANDROID = 10, + XRT_FACE_PARAMETER_INDICES_DIMPLER_R_ANDROID = 11, + XRT_FACE_PARAMETER_INDICES_EYES_CLOSED_L_ANDROID = 12, + XRT_FACE_PARAMETER_INDICES_EYES_CLOSED_R_ANDROID = 13, + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_DOWN_L_ANDROID = 14, + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_DOWN_R_ANDROID = 15, + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_LEFT_L_ANDROID = 16, + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_LEFT_R_ANDROID = 17, + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_RIGHT_L_ANDROID = 18, + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_RIGHT_R_ANDROID = 19, + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_UP_L_ANDROID = 20, + XRT_FACE_PARAMETER_INDICES_EYES_LOOK_UP_R_ANDROID = 21, + XRT_FACE_PARAMETER_INDICES_INNER_BROW_RAISER_L_ANDROID = 22, + XRT_FACE_PARAMETER_INDICES_INNER_BROW_RAISER_R_ANDROID = 23, + XRT_FACE_PARAMETER_INDICES_JAW_DROP_ANDROID = 24, + XRT_FACE_PARAMETER_INDICES_JAW_SIDEWAYS_LEFT_ANDROID = 25, + XRT_FACE_PARAMETER_INDICES_JAW_SIDEWAYS_RIGHT_ANDROID = 26, + XRT_FACE_PARAMETER_INDICES_JAW_THRUST_ANDROID = 27, + XRT_FACE_PARAMETER_INDICES_LID_TIGHTENER_L_ANDROID = 28, + XRT_FACE_PARAMETER_INDICES_LID_TIGHTENER_R_ANDROID = 29, + XRT_FACE_PARAMETER_INDICES_LIP_CORNER_DEPRESSOR_L_ANDROID = 30, + XRT_FACE_PARAMETER_INDICES_LIP_CORNER_DEPRESSOR_R_ANDROID = 31, + XRT_FACE_PARAMETER_INDICES_LIP_CORNER_PULLER_L_ANDROID = 32, + XRT_FACE_PARAMETER_INDICES_LIP_CORNER_PULLER_R_ANDROID = 33, + XRT_FACE_PARAMETER_INDICES_LIP_FUNNELER_LB_ANDROID = 34, + XRT_FACE_PARAMETER_INDICES_LIP_FUNNELER_LT_ANDROID = 35, + XRT_FACE_PARAMETER_INDICES_LIP_FUNNELER_RB_ANDROID = 36, + XRT_FACE_PARAMETER_INDICES_LIP_FUNNELER_RT_ANDROID = 37, + XRT_FACE_PARAMETER_INDICES_LIP_PRESSOR_L_ANDROID = 38, + XRT_FACE_PARAMETER_INDICES_LIP_PRESSOR_R_ANDROID = 39, + XRT_FACE_PARAMETER_INDICES_LIP_PUCKER_L_ANDROID = 40, + XRT_FACE_PARAMETER_INDICES_LIP_PUCKER_R_ANDROID = 41, + XRT_FACE_PARAMETER_INDICES_LIP_STRETCHER_L_ANDROID = 42, + XRT_FACE_PARAMETER_INDICES_LIP_STRETCHER_R_ANDROID = 43, + XRT_FACE_PARAMETER_INDICES_LIP_SUCK_LB_ANDROID = 44, + XRT_FACE_PARAMETER_INDICES_LIP_SUCK_LT_ANDROID = 45, + XRT_FACE_PARAMETER_INDICES_LIP_SUCK_RB_ANDROID = 46, + XRT_FACE_PARAMETER_INDICES_LIP_SUCK_RT_ANDROID = 47, + XRT_FACE_PARAMETER_INDICES_LIP_TIGHTENER_L_ANDROID = 48, + XRT_FACE_PARAMETER_INDICES_LIP_TIGHTENER_R_ANDROID = 49, + XRT_FACE_PARAMETER_INDICES_LIPS_TOWARD_ANDROID = 50, + XRT_FACE_PARAMETER_INDICES_LOWER_LIP_DEPRESSOR_L_ANDROID = 51, + XRT_FACE_PARAMETER_INDICES_LOWER_LIP_DEPRESSOR_R_ANDROID = 52, + XRT_FACE_PARAMETER_INDICES_MOUTH_LEFT_ANDROID = 53, + XRT_FACE_PARAMETER_INDICES_MOUTH_RIGHT_ANDROID = 54, + XRT_FACE_PARAMETER_INDICES_NOSE_WRINKLER_L_ANDROID = 55, + XRT_FACE_PARAMETER_INDICES_NOSE_WRINKLER_R_ANDROID = 56, + XRT_FACE_PARAMETER_INDICES_OUTER_BROW_RAISER_L_ANDROID = 57, + XRT_FACE_PARAMETER_INDICES_OUTER_BROW_RAISER_R_ANDROID = 58, + XRT_FACE_PARAMETER_INDICES_UPPER_LID_RAISER_L_ANDROID = 59, + XRT_FACE_PARAMETER_INDICES_UPPER_LID_RAISER_R_ANDROID = 60, + XRT_FACE_PARAMETER_INDICES_UPPER_LIP_RAISER_L_ANDROID = 61, + XRT_FACE_PARAMETER_INDICES_UPPER_LIP_RAISER_R_ANDROID = 62, + XRT_FACE_PARAMETER_INDICES_TONGUE_OUT_ANDROID = 63, + XRT_FACE_PARAMETER_INDICES_TONGUE_LEFT_ANDROID = 64, + XRT_FACE_PARAMETER_INDICES_TONGUE_RIGHT_ANDROID = 65, + XRT_FACE_PARAMETER_INDICES_TONGUE_UP_ANDROID = 66, + XRT_FACE_PARAMETER_INDICES_TONGUE_DOWN_ANDROID = 67, +}; + +enum xrt_face_tracking_state_android +{ + XRT_FACE_TRACKING_STATE_PAUSED_ANDROID = 0, + XRT_FACE_TRACKING_STATE_STOPPED_ANDROID = 1, + XRT_FACE_TRACKING_STATE_TRACKING_ANDROID = 2, +}; + +#define XRT_FACE_PARAMETER_COUNT_ANDROID 68 + +#define XRT_FACE_REGION_CONFIDENCE_COUNT_ANDROID 3 + struct xrt_facial_base_expression_set_htc { int64_t sample_time_ns; @@ -1796,6 +1887,17 @@ struct xrt_facial_expression_set2_fb bool is_eye_following_blendshapes_valid; }; +struct xrt_facial_expression_set_android +{ + // ordered by xrt_face_parameter_indices_android + float parameters[XRT_FACE_PARAMETER_COUNT_ANDROID]; + float region_confidences[XRT_FACE_REGION_CONFIDENCE_COUNT_ANDROID]; + + uint64_t sample_time_ns; + + XRT_ALIGNAS(8) bool is_valid; +}; + struct xrt_facial_expression_set { union { @@ -1803,6 +1905,7 @@ struct xrt_facial_expression_set struct xrt_facial_eye_expression_set_htc eye_expression_set_htc; struct xrt_facial_lip_expression_set_htc lip_expression_set_htc; struct xrt_facial_expression_set2_fb face_expression_set2_fb; + struct xrt_facial_expression_set_android face_expression_set_android; }; }; diff --git a/src/xrt/include/xrt/xrt_device.h b/src/xrt/include/xrt/xrt_device.h index b36a655e5..0c50ccba2 100644 --- a/src/xrt/include/xrt/xrt_device.h +++ b/src/xrt/include/xrt/xrt_device.h @@ -405,6 +405,16 @@ struct xrt_device int64_t at_timestamp_ns, struct xrt_facial_expression_set *out_value); + /*! + * @brief Gets the face tracking calibration state + * + * @param[in] xdev The device. + * @param[in] out_value Is face tracking calibrated? + * + * @see xrt_input_name + */ + xrt_result_t (*get_face_calibration_state_android)(struct xrt_device *xdev, bool *out_face_is_calibrated); + /*! * @brief Get the body skeleton in T-pose, used to query the skeleton hierarchy, scale, proportions etc * @@ -755,6 +765,19 @@ xrt_device_get_face_tracking(struct xrt_device *xdev, return xdev->get_face_tracking(xdev, facial_expression_type, at_timestamp_ns, out_value); } +/*! + * Helper function for @ref xrt_device::get_face_calibration_state_android. + * + * @copydoc xrt_device::get_face_calibration_state_android + * + * @public @memberof xrt_device + */ +static inline xrt_result_t +xrt_device_get_face_calibration_state_android(struct xrt_device *xdev, bool *out_face_is_calibrated) +{ + return xdev->get_face_calibration_state_android(xdev, out_face_is_calibrated); +} + /*! * Helper function for @ref xrt_device::get_body_skeleton. * diff --git a/src/xrt/ipc/server/ipc_server_handler.c b/src/xrt/ipc/server/ipc_server_handler.c index 4e83fa123..fee7d02c6 100644 --- a/src/xrt/ipc/server/ipc_server_handler.c +++ b/src/xrt/ipc/server/ipc_server_handler.c @@ -2571,6 +2571,17 @@ ipc_handle_device_get_face_tracking(volatile struct ipc_client_state *ics, return xrt_device_get_face_tracking(xdev, facial_expression_type, at_timestamp_ns, out_value); } +xrt_result_t +ipc_handle_device_device_get_face_calibration_state_android(volatile struct ipc_client_state *ics, + uint32_t id, + bool *out_face_is_calibrated) +{ + const uint32_t device_id = id; + struct xrt_device *xdev = NULL; + GET_XDEV_OR_RETURN(ics, device_id, xdev); + return xrt_device_get_face_calibration_state_android(xdev, out_face_is_calibrated); +} + xrt_result_t ipc_handle_device_get_body_skeleton(volatile struct ipc_client_state *ics, uint32_t id, diff --git a/src/xrt/ipc/shared/proto/50-device.json b/src/xrt/ipc/shared/proto/50-device.json index 6037547c1..a7022db84 100644 --- a/src/xrt/ipc/shared/proto/50-device.json +++ b/src/xrt/ipc/shared/proto/50-device.json @@ -181,6 +181,15 @@ ] }, + "device_device_get_face_calibration_state_android": { + "in": [ + {"name": "id", "type": "uint32_t"} + ], + "out": [ + {"name": "face_is_calibrated", "type": "bool"} + ] + }, + "device_get_body_skeleton": { "in": [ {"name": "id", "type": "uint32_t"}, diff --git a/src/xrt/state_trackers/oxr/oxr_face_tracking_android.c b/src/xrt/state_trackers/oxr/oxr_face_tracking_android.c index e79b7036d..e5063a94b 100644 --- a/src/xrt/state_trackers/oxr/oxr_face_tracking_android.c +++ b/src/xrt/state_trackers/oxr/oxr_face_tracking_android.c @@ -14,6 +14,8 @@ #include "oxr_objects.h" #include "oxr_logger.h" #include "oxr_handle.h" +#include "oxr_xret.h" +#include "oxr_two_call.h" static XrResult oxr_face_tracker_android_destroy_cb(struct oxr_logger *log, struct oxr_handle_base *hb) @@ -29,6 +31,29 @@ oxr_face_tracker_android_create(struct oxr_logger *log, const XrFaceTrackerCreateInfoANDROID *createInfo, XrFaceTrackerANDROID *faceTracker) { + bool supported = false; + oxr_system_get_face_tracking_android_support(log, sess->sys->inst, &supported); + if (!supported) { + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "System does not support Android face tracking"); + } + + struct xrt_device *xdev = GET_XDEV_BY_ROLE(sess->sys, face); + if (xdev == NULL) { + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "No device found for face tracking role"); + } + + if (!xdev->supported.face_tracking) { + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "Device does not support HTC facial tracking"); + } + + struct oxr_face_tracker_android *face_tracker_android = NULL; + OXR_ALLOCATE_HANDLE_OR_RETURN(log, face_tracker_android, OXR_XR_DEBUG_FTRACKER, + oxr_face_tracker_android_destroy_cb, &sess->handle); + + face_tracker_android->sess = sess; + face_tracker_android->xdev = xdev; + + *faceTracker = oxr_face_tracker_android_to_openxr(face_tracker_android); return oxr_session_success_result(sess); } @@ -38,6 +63,58 @@ oxr_get_face_state_android(struct oxr_logger *log, const XrFaceStateGetInfoANDROID *getInfo, XrFaceStateANDROID *faceStateOutput) { + /*! + * OXR_TWO_CALL_CHECK_* macro usage here is not technically necessary because validation + * is handled in the API layer before this function is called, but we still re-use them here + * for declarative purposes of handling two-call patterns of setting capacities on the first call. + */ + + /*! + * Use the goto macro variant because a two-call check needs to happen with region confidences + * as well; we can't early exit for only one of them. + */ + XrResult xres = XR_SUCCESS; + OXR_TWO_CALL_CHECK_GOTO(log, faceStateOutput->parametersCapacityInput, + (&faceStateOutput->parametersCountOutput), XRT_FACE_PARAMETER_COUNT_ANDROID, xres, + region_confidences_check); +region_confidences_check: + if (xres != XR_SUCCESS) { + return xres; + } + + OXR_TWO_CALL_CHECK_ONLY(log, faceStateOutput->regionConfidencesCapacityInput, + (&faceStateOutput->regionConfidencesCountOutput), + XRT_FACE_REGION_CONFIDENCE_COUNT_ANDROID, xres); + + const struct oxr_instance *inst = facial_tracker_android->sess->sys->inst; + const int64_t at_timestamp_ns = time_state_ts_to_monotonic_ns(inst->timekeeping, getInfo->time); + + struct xrt_facial_expression_set facial_expression_set_result = {0}; + const xrt_result_t xret = + xrt_device_get_face_tracking(facial_tracker_android->xdev, XRT_INPUT_ANDROID_FACE_TRACKING, at_timestamp_ns, + &facial_expression_set_result); + OXR_CHECK_XRET(log, facial_tracker_android->sess, xret, "oxr_get_face_state_android"); + + const struct xrt_facial_expression_set_android *face_expression_set_android = + &facial_expression_set_result.face_expression_set_android; + + faceStateOutput->isValid = face_expression_set_android->is_valid; + if (faceStateOutput->isValid == XR_FALSE) { + return XR_SUCCESS; + } + + faceStateOutput->sampleTime = + time_state_monotonic_to_ts_ns(inst->timekeeping, face_expression_set_android->sample_time_ns); + + if (faceStateOutput->parametersCapacityInput) { + memcpy(faceStateOutput->parameters, face_expression_set_android->parameters, + sizeof(float) * faceStateOutput->parametersCapacityInput); + } + + if (faceStateOutput->regionConfidencesCapacityInput) { + memcpy(faceStateOutput->regionConfidences, face_expression_set_android->region_confidences, + sizeof(float) * faceStateOutput->regionConfidencesCapacityInput); + } return oxr_session_success_result(facial_tracker_android->sess); } @@ -47,5 +124,12 @@ oxr_get_face_calibration_state_android(struct oxr_logger *log, struct oxr_face_tracker_android *facial_tracker_android, XrBool32 *faceIsCalibratedOutput) { + bool face_is_calibrated = false; + const xrt_result_t xret = + xrt_device_get_face_calibration_state_android(facial_tracker_android->xdev, &face_is_calibrated); + OXR_CHECK_XRET(log, facial_tracker_android->sess, xret, "oxr_get_face_calibration_state_android"); + + *faceIsCalibratedOutput = face_is_calibrated; + return oxr_session_success_result(facial_tracker_android->sess); } diff --git a/src/xrt/state_trackers/oxr/oxr_objects.h b/src/xrt/state_trackers/oxr/oxr_objects.h index be43691a4..1587be653 100644 --- a/src/xrt/state_trackers/oxr/oxr_objects.h +++ b/src/xrt/state_trackers/oxr/oxr_objects.h @@ -1123,6 +1123,9 @@ oxr_system_get_eye_gaze_support(struct oxr_logger *log, struct oxr_instance *ins bool oxr_system_get_force_feedback_support(struct oxr_logger *log, struct oxr_instance *inst); +void +oxr_system_get_face_tracking_android_support(struct oxr_logger *log, struct oxr_instance *inst, bool *supported); + void oxr_system_get_face_tracking_htc_support(struct oxr_logger *log, struct oxr_instance *inst, diff --git a/src/xrt/state_trackers/oxr/oxr_system.c b/src/xrt/state_trackers/oxr/oxr_system.c index 1b3ce1a07..3644ed039 100644 --- a/src/xrt/state_trackers/oxr/oxr_system.c +++ b/src/xrt/state_trackers/oxr/oxr_system.c @@ -405,6 +405,28 @@ oxr_system_get_force_feedback_support(struct oxr_logger *log, struct oxr_instanc return false; } +void +oxr_system_get_face_tracking_android_support(struct oxr_logger *log, struct oxr_instance *inst, bool *supported) +{ + assert(supported); + + *supported = false; + struct oxr_system *sys = &inst->system; + const struct xrt_device *face_xdev = GET_XDEV_BY_ROLE(sys, face); + + if (face_xdev == NULL || !face_xdev->supported.face_tracking || face_xdev->inputs == NULL) { + return; + } + + for (size_t input_idx = 0; input_idx < face_xdev->input_count; ++input_idx) { + const struct xrt_input *input = &face_xdev->inputs[input_idx]; + if (input->name == XRT_INPUT_ANDROID_FACE_TRACKING) { + *supported = true; + return; + } + } +} + void oxr_system_get_face_tracking_htc_support(struct oxr_logger *log, struct oxr_instance *inst, @@ -568,6 +590,20 @@ oxr_system_get_properties(struct oxr_logger *log, struct oxr_system *sys, XrSyst } #endif +#ifdef OXR_HAVE_ANDROID_face_tracking + XrSystemFaceTrackingPropertiesANDROID *android_face_tracking_props = NULL; + if (sys->inst->extensions.ANDROID_face_tracking) { + android_face_tracking_props = OXR_GET_OUTPUT_FROM_CHAIN( + properties, XR_TYPE_SYSTEM_FACE_TRACKING_PROPERTIES_ANDROID, XrSystemFaceTrackingPropertiesANDROID); + } + + if (android_face_tracking_props) { + bool supported = false; + oxr_system_get_face_tracking_android_support(log, sys->inst, &supported); + android_face_tracking_props->supportsFaceTracking = supported; + } +#endif // OXR_HAVE_HTC_facial_tracking + #ifdef OXR_HAVE_HTC_facial_tracking XrSystemFacialTrackingPropertiesHTC *htc_facial_tracking_props = NULL; if (sys->inst->extensions.HTC_facial_tracking) { -- 2.51.2