three.jsを使用して360度画像を見ることができるウェブページを作ろうとしています。
途中まではうまくいっていたのですが、スマートフォンの傾きと連動させてカメラの向きを変えようとする部分で行き詰ってしまいました。
そもそも行列やベクトルの概念に疎く（勉強中ではありますが）、three.jsのそれぞれのメソッド（）の働きについてもいろいろ試してみてたまたまうまくいっているような状態です。

以下私が出来たことと出来なかったこと、実装コードを記載いたします。なにかヒントでも頂ければ幸いです。よろしくお願いいたします。

出来たこと

XYZ空間の原点を中心として SphereGeometry を使用したメッシュを配置し、360度画像をメッシュの内側に描画
XYZ空間の原点に PerspectiveCamera を配置し、レンダラーを通して Canvas にカメラに写っているメッシュを表示
mousedown, mouseup, mousemoveなどのイベントから、マウスやタップの位置を利用してカメラの向きを変更
deviceorientation のイベントから、スマホの向きを利用してカメラの向きを変更

出来なかったこと

出来たこと3および4はそれぞれ単独で動作するが、連動して動作しない
（タップでカメラの向きを変更しても、指を話すとスマホの向きにカメラが戻ってしまう）
（カメラが0時の方向を向いた状態で、タップでカメラを3時の方向に向けても、指を離すと0時の方向に戻ってしまう）
（期待動作としては、上記の状態で指を離してもカメラの向きが3時のままで、そこを起点としてさらにスマホの向きでカメラの向きを変更できるようにしたい）

実装コード

一応全て載せましたが、コアの部分は add-gestures.js です。

index.html

<!doctype html>
<html>
  <head>
    <meta name="viewport" content="width=device-width,initial-scale=1">

    <link rel="stylesheet" href="pages/vr/style.css') ?>
  </head>

  <body>
    <canvas id="vr-canvas"></canvas>

    <script src="/js/lib/threejs/r104/build/three.min.js"></script>

    <script src="/js/pages/vr/init-vr.js"></script>
    <script src="/js/pages/vr/add-gestures.js"></script>
    <script src="/js/pages/vr/add-sphere.js"></script>
  </body>
</html>

init-vr.js

window.VRApp = window.VRApp || {};

const canvas = document.querySelector("#vr-canvas");

const renderer = (() => {
  const webGLRenderer = new THREE.WebGLRenderer({ canvas });

  webGLRenderer.setPixelRatio(window.devicePixelRatio);

  return webGLRenderer;
})();

const scene = new THREE.Scene();

const camera = (() => {
  const perspectiveCamera = new THREE.PerspectiveCamera(100, canvas.width / canvas.height, 0.01, 100);

  perspectiveCamera.rotation.order = "ZYX";

  return perspectiveCamera;
})();

const animate = () => {
  requestAnimationFrame(animate);

  renderer.render(scene, camera);
};

animate();

window.VRApp.renderer = renderer;
window.VRApp.scene = scene;
window.VRApp.camera = camera;

add-gestures.js

window.VRApp = window.VRApp || {};

const State = {
  Neutral: 0x0000,
  RotateCamera: 0x0001,
};

let state = State.Neutral;

let windowOrientation = window.orientation || 0;
let cameraRotationCache = window.VRApp.camera.rotation.clone();

let mousePositionCache = {
  x: 0,
  y: 0,
  minYDiff: 0,
  maxYDiff: 0,
};

const setState = (newState) => {
  if (State.hasOwnProperty(newState)) {
    state = State[newState];
  }
};

const checkState = (targetState) => {
  if (State.hasOwnProperty(targetState)) {
    return state === State[targetState];
  }

  return false;
};

const setQuaternion = (() => {
  const zee = new THREE.Vector3(0, 0, 1);
  const euler = new THREE.Euler();
  const q0 = new THREE.Quaternion();
  const q1 = new THREE.Quaternion(-1 * Math.sqrt(0.5), 0, 0, Math.sqrt(0.5));

  return (alpha, beta, gamma, orientation) => {
    euler.set(beta, alpha, -1 * gamma, "YXZ");

    window.VRApp.camera.quaternion.setFromEuler(euler);
    window.VRApp.camera.quaternion.multiply(q1);
    window.VRApp.camera.quaternion.multiply(q0.setFromAxisAngle(zee, -1 * orientation));
  };
})();

const onMouseDownHandler = (clientX, clientY) => {
  setState("RotateCamera");

  cameraRotationCache = window.VRApp.camera.rotation.clone();

  mousePositionCache.x = clientX;
  mousePositionCache.y = clientY;
  mousePositionCache.minYDiff = -90 - (cameraRotationCache.x * (180 / Math.PI)) - (clientY * (Math.PI / 180));
  mousePositionCache.maxYDiff = 90 - (cameraRotationCache.x * (180 / Math.PI)) - (clientY * (Math.PI / 180));
};

const onMouseMoveHandler = (clientX, clientY) => {
  if (checkState("RotateCamera")) {
    window.VRApp.camera.rotation.order = "ZYX";

    let xDiff = clientX - mousePositionCache.x;
    let yDiff = clientY - mousePositionCache.y;

    if (yDiff < mousePositionCache.minYDiff) {
      yDiff = mousePositionCache.minYDiff;

      mousePositionCache.y = clientY - mousePositionCache.minYDiff;
    }

    if (yDiff > mousePositionCache.maxYDiff) {
      yDiff = mousePositionCache.maxYDiff;

      mousePositionCache.y = clientY - mousePositionCache.maxYDiff;
    }

    let newAngleX = cameraRotationCache.x + (yDiff * (Math.PI / 180));
    let newAngleY = cameraRotationCache.y + (xDiff * (Math.PI / 180));

    window.VRApp.camera.rotation.x = newAngleX;
    window.VRApp.camera.rotation.y = newAngleY;
  }
};

const onMouseUpHandler = () => {
  setState("Neutral");

  cameraRotationCache = window.VRApp.camera.rotation.clone();

  mousePositionCache.x = 0;
  mousePositionCache.y = 0;
  mousePositionCache.minYDiff = 0;
  mousePositionCache.maxYDiff = 0;
};

if ("onresize" in window) {
  window.addEventListener("resize", (event) => {
    const width = window.innerWidth;
    const height = window.innerHeight;

    window.VRApp.renderer.domElement.width = width;
    window.VRApp.renderer.domElement.height = height;

    window.VRApp.renderer.domElement.style.height = height + "px";

    window.VRApp.renderer.setSize(width, height);

    window.VRApp.camera.aspect = width / height;
    window.VRApp.camera.updateProjectionMatrix();
  });
}

if ("onload" in window) {
  window.addEventListener("load", (event) => {
    const width = window.innerWidth;
    const height = window.innerHeight;

    window.VRApp.renderer.domElement.width = width;
    window.VRApp.renderer.domElement.height = height;

    window.VRApp.renderer.domElement.style.height = height + "px";

    window.VRApp.renderer.setSize(width, height);

    window.VRApp.camera.aspect = width / height;
    window.VRApp.camera.updateProjectionMatrix();
  });
}

if ("onmousedown" in window.VRApp.renderer.domElement) {
  window.VRApp.renderer.domElement.addEventListener("mousedown", (event) => {
    onMouseDownHandler(event.clientX, event.clientY);
  });
}

if ("onmousemove" in window.VRApp.renderer.domElement) {
  window.VRApp.renderer.domElement.addEventListener("mousemove", (event) => {
    onMouseMoveHandler(event.clientX, event.clientY);
  });
}

if ("onmouseup" in window.VRApp.renderer.domElement) {
  window.VRApp.renderer.domElement.addEventListener("mouseup", (event) => {
    onMouseUpHandler();
  });
}

if ("onmouseleave" in window.VRApp.renderer.domElement) {
  window.VRApp.renderer.domElement.addEventListener("mouseleave", (event) => {
    onMouseUpHandler();
  });
}

if ("ontouchstart" in window.VRApp.renderer.domElement) {
  window.VRApp.renderer.domElement.addEventListener("touchstart", (event) => {
    event.preventDefault();

    if (event.touches.length === 1) {
      const touch = event.touches[0];

      onMouseDownHandler(touch.clientX, touch.clientY);
    }
  });
}

if ("ontouchmove" in window.VRApp.renderer.domElement) {
  window.VRApp.renderer.domElement.addEventListener("touchmove", (event) => {
    event.preventDefault();

    if (event.touches.length === 1) {
      const touch = event.touches[0];

      onMouseMoveHandler(touch.clientX, touch.clientY);
    }
  });
}

if ("ontouchend" in window.VRApp.renderer.domElement) {
  window.VRApp.renderer.domElement.addEventListener("touchend", (event) => {
    event.preventDefault();

    onMouseUpHandler();
  });
}

if ("ontouchcancel" in window.VRApp.renderer.domElement) {
  window.VRApp.renderer.domElement.addEventListener("touchcancel", (event) => {
    event.preventDefault();

    onMouseUpHandler();
  });
}

if ("onorientationchange" in window) {
  window.addEventListener("orientationchange", (event) => {
    windowOrientation = window.orientation || 0;
  });
}

if ("ondeviceorientation" in window) {
  window.addEventListener("deviceorientation", (event) => {
    if (checkState("Neutral")) {
      let alpha = event.alpha * (Math.PI / 180);
      let beta = event.beta * (Math.PI / 180);
      let gamma = event.gamma * (Math.PI / 180);
      let orientation = windowOrientation * (Math.PI / 180);

      setQuaternion(alpha, beta, gamma, orientation);
    }
  });
}

add-sphere.js

window.VRApp = window.VRApp || {};

const sphere = (() => {
  const geometry = new THREE.SphereGeometry(100, 64, 64);

  geometry.scale(1, 1, -1);
  geometry.rotateY(Math.PI / 2);

  const material = new THREE.MeshBasicMaterial({
  });

  const mesh = new THREE.Mesh(geometry, material);

  return mesh;
})();

const textureLoader = new THREE.TextureLoader();
const texture = textureLoader.load("/img/pages/vr/sample-360.jpg");

sphere.material.map = texture;

window.VRApp.scene.add(sphere);