diff --git a/doc/changes/auxiliary/mr.350.md b/doc/changes/auxiliary/mr.350.md new file mode 100644 index 000000000..cfc616630 --- /dev/null +++ b/doc/changes/auxiliary/mr.350.md @@ -0,0 +1,3 @@ +math: Correctly validate quaternion using non-squared "length" instead of +squared "length", certain combinations of elements would produce valid regular +"length" but not valid squared ones. diff --git a/src/xrt/auxiliary/math/m_base.cpp b/src/xrt/auxiliary/math/m_base.cpp index 44e2877bd..7c1aa4d3d 100644 --- a/src/xrt/auxiliary/math/m_base.cpp +++ b/src/xrt/auxiliary/math/m_base.cpp @@ -151,7 +151,17 @@ math_quat_validate(const struct xrt_quat *quat) auto rot = copy(*quat); const float FLOAT_EPSILON = Eigen::NumTraits::epsilon(); - auto norm = rot.squaredNorm(); + /* + * This was originally squaredNorm, but that could result in a norm + * value that was further from 1.0f then FLOAT_EPSILON (two). + * + * Our tracking system would produce such orientations and looping those + * back into say a quad layer would cause this to fail. And even + * normalizing the quat would not fix this as normalizations uses + * non-squared "length" which does fall into the range and doesn't + * change the elements of the quat. + */ + auto norm = rot.norm(); if (norm > 1.0f + FLOAT_EPSILON || norm < 1.0f - FLOAT_EPSILON) { return false; }