Skip to content

Commit

Permalink
Use fillPoses and fillJointRadii instead of getJointPose (#5298)
Browse files Browse the repository at this point in the history 
* Don't offset reference space for hand-tracking-controls

* Use fillPoses and fillJointRadii once per tick in hand-tracking-controls

* Update hand-tracking-controls test to call tick before detectPinch

---------

Co-authored-by: Noeri Huisman <mrxz@users.noreply.github.com>
  • Loading branch information
@mrxz
mrxz and mrxz authored Jun 14, 2023
1 parent 61bff0d commit 0272f92
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 75 deletions.
107 changes: 51 additions & 56 deletions src/components/hand-tracking-controls.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* global THREE, XRRigidTransform */
/* global THREE */
var registerComponent = require('../core/component').registerComponent;
var bind = require('../utils/bind');

Expand Down Expand Up @@ -36,6 +36,9 @@ var JOINTS = [
'pinky-finger-tip'
];

var THUMB_TIP_INDEX = 4;
var INDEX_TIP_INDEX = 9;

var PINCH_START_DISTANCE = 0.015;
var PINCH_END_DISTANCE = 0.03;
var PINCH_POSITION_INTERPOLATION = 0.5;
Expand Down Expand Up @@ -82,6 +85,10 @@ module.exports.Component = registerComponent('hand-tracking-controls', {
this.pinchEventDetail = {position: new THREE.Vector3()};
this.indexTipPosition = new THREE.Vector3();

this.hasPoses = false;
this.jointPoses = new Float32Array(16 * JOINTS.length);
this.jointRadii = new Float32Array(JOINTS.length);

this.bindMethods();

this.updateReferenceSpace = this.updateReferenceSpace.bind(this);
Expand All @@ -96,7 +103,7 @@ module.exports.Component = registerComponent('hand-tracking-controls', {
if (!xrSession) { return; }
var referenceSpaceType = self.el.sceneEl.systems.webxr.sessionReferenceSpaceType;
xrSession.requestReferenceSpace(referenceSpaceType).then(function (referenceSpace) {
self.referenceSpace = referenceSpace.getOffsetReferenceSpace(new XRRigidTransform({x: 0, y: 1.5, z: 0}));
self.referenceSpace = referenceSpace;
}).catch(function (error) {
self.el.sceneEl.systems.webxr.warnIfFeatureNotRequested(referenceSpaceType, 'tracked-controls-webxr uses reference space ' + referenceSpaceType);
throw error;
Expand All @@ -121,11 +128,17 @@ module.exports.Component = registerComponent('hand-tracking-controls', {
var controller = this.el.components['tracked-controls'] && this.el.components['tracked-controls'].controller;
var frame = sceneEl.frame;
var trackedControlsWebXR = this.el.components['tracked-controls-webxr'];
if (!controller || !frame || !trackedControlsWebXR) { return; }
var referenceSpace = this.referenceSpace;
if (!controller || !frame || !referenceSpace || !trackedControlsWebXR) { return; }
this.hasPoses = false;
if (controller.hand) {
this.el.object3D.position.set(0, 0, 0);
this.el.object3D.rotation.set(0, 0, 0);
if (frame.getJointPose) { this.updateHandModel(); }

this.hasPoses = frame.fillPoses(controller.hand.values(), referenceSpace, this.jointPoses) &&
frame.fillJointRadii(controller.hand.values(), this.jointRadii);

this.updateHandModel();
this.detectGesture();
}
},
Expand All @@ -148,47 +161,44 @@ module.exports.Component = registerComponent('hand-tracking-controls', {
return null;
},

updateHandMeshModel: function () {
var frame = this.el.sceneEl.frame;
var controller = this.el.components['tracked-controls'] && this.el.components['tracked-controls'].controller;
var referenceSpace = this.referenceSpace;

if (!controller || !this.mesh || !referenceSpace) { return; }
this.mesh.visible = false;
for (var inputjoint of controller.hand.values()) {
var bone;
var jointPose;
var jointTransform;
jointPose = frame.getJointPose(inputjoint, referenceSpace);
bone = this.getBone(inputjoint.jointName);
if (bone != null && jointPose) {
jointTransform = jointPose.transform;
this.mesh.visible = true;
bone.position.copy(jointTransform.position);
bone.quaternion.copy(jointTransform.orientation);
updateHandMeshModel: (function () {
var jointPose = new THREE.Matrix4();
return function () {
var jointPoses = this.jointPoses;
var controller = this.el.components['tracked-controls'] && this.el.components['tracked-controls'].controller;
var i = 0;

if (!controller || !this.mesh) { return; }
this.mesh.visible = false;
if (!this.hasPoses) { return; }
for (var inputjoint of controller.hand.values()) {
var bone = this.getBone(inputjoint.jointName);
if (bone != null) {
this.mesh.visible = true;
jointPose.fromArray(jointPoses, i * 16);
bone.position.setFromMatrixPosition(jointPose);
bone.quaternion.setFromRotationMatrix(jointPose);
}
i++;
}
}
},
};
})(),

updateHandDotsModel: function () {
var frame = this.el.sceneEl.frame;
var jointPoses = this.jointPoses;
var jointRadii = this.jointRadii;
var controller = this.el.components['tracked-controls'] && this.el.components['tracked-controls'].controller;
var trackedControlsWebXR = this.el.components['tracked-controls-webxr'];
var referenceSpace = trackedControlsWebXR.system.referenceSpace;
var jointEl;
var object3D;
var jointPose;
var i = 0;

for (var inputjoint of controller.hand.values()) {
jointEl = this.jointEls[i++];
for (var i = 0; i < controller.hand.size; i++) {
jointEl = this.jointEls[i];
object3D = jointEl.object3D;
jointPose = frame.getJointPose(inputjoint, referenceSpace);
jointEl.object3D.visible = !!jointPose;
if (!jointPose) { continue; }
object3D.matrix.elements = jointPose.transform.matrix;
jointEl.object3D.visible = this.hasPoses;
if (!this.hasPoses) { continue; }
object3D.matrix.fromArray(jointPoses, i * 16);
object3D.matrix.decompose(object3D.position, object3D.rotation, object3D.scale);
jointEl.setAttribute('scale', {x: jointPose.radius, y: jointPose.radius, z: jointPose.radius});
jointEl.setAttribute('scale', {x: jointRadii[i], y: jointRadii[i], z: jointRadii[i]});
}
},

Expand All @@ -198,47 +208,32 @@ module.exports.Component = registerComponent('hand-tracking-controls', {

detectPinch: (function () {
var thumbTipPosition = new THREE.Vector3();
var jointPose = new THREE.Matrix4();
return function () {
var frame = this.el.sceneEl.frame;
var indexTipPosition = this.indexTipPosition;
var controller = this.el.components['tracked-controls'] && this.el.components['tracked-controls'].controller;
var trackedControlsWebXR = this.el.components['tracked-controls-webxr'];
var referenceSpace = this.referenceSpace || trackedControlsWebXR.system.referenceSpace;
var indexTip = controller.hand.get('index-finger-tip');
var thumbTip = controller.hand.get('thumb-tip');
if (!indexTip ||
!thumbTip) { return; }
var indexTipPose = frame.getJointPose(indexTip, referenceSpace);
var thumbTipPose = frame.getJointPose(thumbTip, referenceSpace);
if (!this.hasPoses) { return; }

if (!indexTipPose || !thumbTipPose) { return; }

thumbTipPosition.copy(thumbTipPose.transform.position);
indexTipPosition.copy(indexTipPose.transform.position);
thumbTipPosition.setFromMatrixPosition(jointPose.fromArray(this.jointPoses, THUMB_TIP_INDEX * 16));
indexTipPosition.setFromMatrixPosition(jointPose.fromArray(this.jointPoses, INDEX_TIP_INDEX * 16));

var distance = indexTipPosition.distanceTo(thumbTipPosition);

if (distance < PINCH_START_DISTANCE && this.isPinched === false) {
this.isPinched = true;
this.pinchEventDetail.position.copy(indexTipPosition).lerp(thumbTipPosition, PINCH_POSITION_INTERPOLATION);
this.pinchEventDetail.position.y += 1.5;
this.el.emit('pinchstarted', this.pinchEventDetail);
}

if (distance > PINCH_END_DISTANCE && this.isPinched === true) {
this.isPinched = false;
this.pinchEventDetail.position.copy(indexTipPosition).lerp(thumbTipPosition, PINCH_POSITION_INTERPOLATION);
this.pinchEventDetail.position.y += 1.5;
this.el.emit('pinchended', this.pinchEventDetail);
}

if (this.isPinched) {
this.pinchEventDetail.position.copy(indexTipPosition).lerp(thumbTipPosition, PINCH_POSITION_INTERPOLATION);
this.pinchEventDetail.position.y += 1.5;
this.el.emit('pinchmoved', this.pinchEventDetail);
}

indexTipPosition.y += 1.5;
};
})(),

Expand Down Expand Up @@ -314,7 +309,7 @@ module.exports.Component = registerComponent('hand-tracking-controls', {
if (!this.skinnedMesh) { return; }
this.bones = skinnedMesh.skeleton.bones;
this.el.removeObject3D('mesh');
mesh.position.set(0, 1.5, 0);
mesh.position.set(0, 0, 0);
mesh.rotation.set(0, 0, 0);
skinnedMesh.frustumCulled = false;
skinnedMesh.material = new THREE.MeshStandardMaterial({skinning: true, color: this.data.modelColor});
Expand Down
44 changes: 25 additions & 19 deletions tests/components/hand-tracking-controls.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ suite('tracked-controls-webxr', function () {
var el;
var system;
var standingMatrix = new THREE.Matrix4();
var index = {transform: {position: {x: 0, y: 0, z: 0}}};
var thumb = {transform: {position: {x: 0, y: 0, z: 0}}};
var indexMatrix = new THREE.Matrix4();
var thumbMatrix = new THREE.Matrix4();
var THUMB_TIP_INDEX = 4;
var INDEX_TIP_INDEX = 9;
var indexPosition = new THREE.Vector3();
var thumbPosition = new THREE.Vector3();
var thumbObj = {};
Expand All @@ -18,15 +20,15 @@ suite('tracked-controls-webxr', function () {
el = entityFactory();
setTimeout(() => {
el.sceneEl.addEventListener('loaded', function () {
window.XRHand = {
INDEX_PHALANX_TIP: 0,
THUMB_PHALANX_TIP: 1
};
el.sceneEl.hasWebXR = true;
el.sceneEl.frame = {
getJointPose: function (joint, fingerPose) {
var transform = joint === thumbObj ? thumb : index;
return transform;
fillPoses: function (joints, referenceSpace, array) {
thumbMatrix.toArray(array, 16 * THUMB_TIP_INDEX);
indexMatrix.toArray(array, 16 * INDEX_TIP_INDEX);
return true;
},
fillJointRadii: function () {
return true;
}
};
system = el.sceneEl.systems['tracked-controls-webxr'];
Expand All @@ -37,11 +39,18 @@ suite('tracked-controls-webxr', function () {
get: function (joint) {
var jointObject = joint === 'thumb-tip' ? thumbObj : indexObj;
return jointObject;
},
values: function () {
return [
{ jointName: 'thumb-tip' },
{ jointName: 'index-finger-tip' }
];
}
}
};
system.controllers = [controller];
el.setAttribute('hand-tracking-controls', {hand: 'left'});
el.components['hand-tracking-controls'].referenceSpace = {};
done();
});
});
Expand All @@ -60,13 +69,12 @@ suite('tracked-controls-webxr', function () {
test('pinchstarted', function () {
const emitSpy = sinon.spy(el, 'emit');
el.setAttribute('hand-tracking-controls', {hand: 'left'});
el.components['hand-tracking-controls'].tick();
el.components['hand-tracking-controls'].checkIfControllerPresent();
el.components['hand-tracking-controls'].detectPinch();
assert.equal(emitSpy.getCalls()[0].args[0], 'pinchstarted');
indexPosition.copy(index.transform.position);
indexPosition.y += 1.5;
thumbPosition.copy(thumb.transform.position);
thumbPosition.y += 1.5;
indexPosition.setFromMatrixPosition(indexMatrix);
thumbPosition.setFromMatrixPosition(thumbMatrix);
const indexThumbDistance = indexPosition.distanceTo(thumbPosition);
assert.isAtMost(emitSpy.getCalls()[0].args[1].position.distanceTo(indexPosition), indexThumbDistance);
assert.isAtMost(emitSpy.getCalls()[0].args[1].position.distanceTo(thumbPosition), indexThumbDistance);
Expand All @@ -77,14 +85,12 @@ suite('tracked-controls-webxr', function () {
el.setAttribute('hand-tracking-controls', {hand: 'left'});
el.components['hand-tracking-controls'].checkIfControllerPresent();
el.components['hand-tracking-controls'].isPinched = true;
thumb.transform.position.z = 10;
thumbPosition.copy(thumb.transform.position);
thumbMatrix.setPosition(0, 0, 10);
el.components['hand-tracking-controls'].tick();
el.components['hand-tracking-controls'].detectPinch();
assert.equal(emitSpy.getCalls()[0].args[0], 'pinchended');
indexPosition.copy(index.transform.position);
indexPosition.y += 1.5;
thumbPosition.copy(thumb.transform.position);
thumbPosition.y += 1.5;
indexPosition.setFromMatrixPosition(indexMatrix);
thumbPosition.setFromMatrixPosition(thumbMatrix);
const indexThumbDistance = indexPosition.distanceTo(thumbPosition);
assert.isAtMost(emitSpy.getCalls()[0].args[1].position.distanceTo(indexPosition), indexThumbDistance);
assert.isAtMost(emitSpy.getCalls()[0].args[1].position.distanceTo(thumbPosition), indexThumbDistance);
Expand Down

0 comments on commit 0272f92

Please sign in to comment.