Fix camera orientation for scenepointer (#151)

* Fix camera orientation for scenepointer

* Fix raycast coordinate frame

* Fix raycast coordinate frame

---------

Co-authored-by: Brent Yi <yibrenth@gmail.com>

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "viser"
-version = "0.1.15"
+version = "0.1.16"
 description = "3D visualization + Python"
 readme = "README.md"
 license = { text="MIT" }

src/viser/client/src/App.tsx

@@ -49,6 +49,7 @@ import { useSceneTreeState } from "./SceneTreeState";
 import { GetRenderRequestMessage, Message } from "./WebsocketMessages";
 import { makeThrottledMessageSender } from "./WebsocketFunctions";
 import { useDisclosure } from "@mantine/hooks";
+import { computeT_threeworld_world } from "./WorldTransformUtils";
 
 export type ViewerContextContents = {
   // Zustand hooks.
@@ -234,6 +235,7 @@ function ViewerCanvas({ children }: { children: React.ReactNode }) {
         mouseVector.y =
           1 -
           2 * (e.nativeEvent.offsetY / viewer.canvasRef.current!.clientHeight);
+        console.log(mouseVector.x, mouseVector.y);
         if (
           mouseVector.x > 1 ||
           mouseVector.x < -1 ||
@@ -245,19 +247,24 @@ function ViewerCanvas({ children }: { children: React.ReactNode }) {
         const raycaster = new THREE.Raycaster();
         raycaster.setFromCamera(mouseVector, viewer.cameraRef.current!);
 
+        const T_world_threeworld = computeT_threeworld_world(viewer).invert();
+
+        const origin = raycaster.ray.origin
+          .clone()
+          .applyMatrix4(T_world_threeworld);
+
+        // Compute just the rotation term without new memory allocation; this
+        // will mutate T_world_threeworld!
+        const R_world_threeworld = T_world_threeworld.setPosition(0.0, 0.0, 0);
+        const direction = raycaster.ray.direction
+          .clone()
+          .applyMatrix4(R_world_threeworld);
+
         sendClickThrottled({
           type: "ScenePointerMessage",
           event_type: "click",
-          ray_origin: [
-            raycaster.ray.origin.x,
-            -raycaster.ray.origin.z,
-            raycaster.ray.origin.y,
-          ],
-          ray_direction: [
-            raycaster.ray.direction.x,
-            -raycaster.ray.direction.z,
-            raycaster.ray.direction.y,
-          ],
+          ray_origin: [origin.x, origin.y, origin.z],
+          ray_direction: [direction.x, direction.y, direction.z],
         });
       }}
     >

src/viser/client/src/CameraControls.tsx

@@ -6,7 +6,7 @@ import * as holdEvent from "hold-event";
 import React, { useContext, useRef } from "react";
 import { PerspectiveCamera } from "three";
 import * as THREE from "three";
-import { getT_threeworld_world } from "./WorldTransformUtils";
+import { computeT_threeworld_world } from "./WorldTransformUtils";
 
 export function SynchronizedCameraControls() {
   const viewer = useContext(ViewerContext)!;
@@ -57,7 +57,7 @@ export function SynchronizedCameraControls() {
     const R_threecam_cam = new THREE.Quaternion().setFromEuler(
       new THREE.Euler(Math.PI, 0.0, 0.0),
     );
-    const T_world_threeworld = getT_threeworld_world(viewer).invert();
+    const T_world_threeworld = computeT_threeworld_world(viewer).invert();
     const T_world_camera = T_world_threeworld.clone()
       .multiply(
         new THREE.Matrix4()

src/viser/client/src/WebsocketInterface.tsx

@@ -28,7 +28,7 @@ import { useFrame } from "@react-three/fiber";
 import GeneratedGuiContainer from "./ControlPanel/Generated";
 import { MantineProvider, Paper, Progress } from "@mantine/core";
 import { IconCheck } from "@tabler/icons-react";
-import { getT_threeworld_world } from "./WorldTransformUtils";
+import { computeT_threeworld_world } from "./WorldTransformUtils";
 
 /** Convert raw RGB color buffers to linear color buffers. **/
 function threeColorBufferFromUint8Buffer(colors: ArrayBuffer) {
@@ -427,7 +427,7 @@ function useMessageHandler() {
       case "SetCameraLookAtMessage": {
         const cameraControls = viewer.cameraControlRef.current!;
 
-        const T_threeworld_world = getT_threeworld_world(viewer);
+        const T_threeworld_world = computeT_threeworld_world(viewer);
         const target = new THREE.Vector3(
           message.look_at[0],
           message.look_at[1],
@@ -440,7 +440,7 @@ function useMessageHandler() {
       case "SetCameraUpDirectionMessage": {
         const camera = viewer.cameraRef.current!;
         const cameraControls = viewer.cameraControlRef.current!;
-        const T_threeworld_world = getT_threeworld_world(viewer);
+        const T_threeworld_world = computeT_threeworld_world(viewer);
         const updir = new THREE.Vector3(
           message.position[0],
           message.position[1],
@@ -478,7 +478,7 @@ function useMessageHandler() {
           message.position[2],
         );
 
-        const T_threeworld_world = getT_threeworld_world(viewer);
+        const T_threeworld_world = computeT_threeworld_world(viewer);
         position_cmd.applyMatrix4(T_threeworld_world);
 
         cameraControls.setPosition(

src/viser/client/src/WorldTransformUtils.ts

@@ -4,7 +4,7 @@ import * as THREE from "three";
 /** Helper for computing the transformation between the three.js world and the
  * Python-exposed world frames. This is useful for things like switching
  * between +Y and +Z up directions for the world frame. */
-export function getT_threeworld_world(viewer: ViewerContextContents) {
+export function computeT_threeworld_world(viewer: ViewerContextContents) {
   const wxyz = viewer.nodeAttributesFromName.current[""]!.wxyz!;
   const position = viewer.nodeAttributesFromName.current[""]!.position ?? [
     0, 0, 0,