Add audio and obras with video projections

packages/client/src/3d/ArtworkCollection.tsx

@@ -9,25 +9,25 @@ import Carolina from './models/obras/Carolina'
 import Julian from './models/obras/Julian'
 import Mateo from './models/obras/Mateo'
 import Gaston from './models/obras/Gaston'
-// import Daniela from './models/obras/Daniela'
-// import Pedro from './models/obras/Pedro'
-// import DiegoGuido from './models/obras/DiegoGuido'
+import Daniela from './models/obras/Daniela'
+import Pedro from './models/obras/Pedro'
+import DiegoGuido from './models/obras/DiegoGuido'
 
 const models: (JSX.Element | null)[] = [
-  null, // <Daniela />,
+  <Daniela key="daniela" />, // <Daniela />,
   null, // Juan cruz
   null, // Lucia cordoba
   null, // Camila
-  <Julieta />,
+  <Julieta key="julieta" />,
   null, // Lucia Novello
-  <Santiaga />,
-  <Carolina />,
+  <Santiaga key="santiaga" />,
+  <Carolina key="carolina" />,
   null, // Edu
-  <Julian />,
-  <Mateo />,
-  <Gaston />,
-  null, //<Pedro />,
-  null, //<DiegoGuido />,
+  <Julian key="julian" />,
+  <Mateo key="mateo" />,
+  <Gaston key="gaston" />,
+  <Pedro key="pedro" />,
+  <DiegoGuido key="diego_guido" />,
 ]
 
 function ArtworkCollection(props: JSX.IntrinsicElements['group']) {

packages/client/src/3d/Collisions.tsx

@@ -6,14 +6,14 @@ import ParkCollisions from './models/Park_collisions'
 import StreetCollisions from './models/Street_collisions'
 import MuseumCollisions from './models/Museum_collisions'
 import EntranceCollsions from './models/Entrance_collisions'
-import { Mesh, MeshBasicMaterial } from 'three'
+import { Mesh } from 'three'
 
 const Model = forwardRef(
   (props: JSX.IntrinsicElements['group'], ref: ForwardedRef<Mesh>) => {
-    const material = new MeshBasicMaterial({
-      wireframe: true,
-      color: 0xffff00,
-    })
+    // const material = new MeshBasicMaterial({
+    //   wireframe: true,
+    //   color: 0xffff00,
+    // })
 
     return (
       <group ref={ref} {...props} dispose={null}>
@@ -26,4 +26,6 @@ const Model = forwardRef(
   },
 )
 
+Model.displayName = 'collisions'
+
 export default Model

packages/client/src/3d/Controls.tsx

@@ -14,7 +14,6 @@ const Controls: React.FC = () => {
       const onLock = () => setPointerLockStatus(true)
       const onUnlock = () => {
         setPointerLockStatus(false)
-        console.log('unlock')
       }
 
       ref.current.addEventListener?.('lock', onLock)

packages/client/src/3d/ObraAudio.tsx

@@ -0,0 +1,36 @@
+import React, { useEffect, useRef } from 'react'
+import { PositionalAudio, Box } from '@react-three/drei'
+import { PositionalAudio as PositionalAudioImpl } from 'three'
+import useStore from '../store'
+
+type Props = JSX.IntrinsicElements['positionalAudio'] & {
+  url: string
+  distance?: number
+  loop?: boolean
+}
+
+const ObraAudio: React.FC<Props> = ({ position, url, loop = true, ...props }) => {
+  const pointerLocked = useStore((state) => state.pointerLocked)
+  const audioRef = useRef<PositionalAudioImpl>()
+
+  useEffect(() => {
+    if (audioRef.current) {
+      if (!pointerLocked && audioRef.current.isPlaying) {
+        audioRef.current.pause()
+      } else if (pointerLocked && !audioRef.current.isPlaying) {
+        audioRef.current.play()
+      }
+
+      audioRef.current.setDirectionalCone(35, 60, 0.1)
+      audioRef.current.setRefDistance(0.5)
+    }
+  }, [audioRef.current, pointerLocked])
+
+  return (
+    <group rotation={[0, -Math.PI / 2, 0]} position={[14, 1.6, 0]}>
+      <PositionalAudio ref={audioRef} url={url} loop={loop} autoplay={false} {...props} />
+    </group>
+  )
+}
+
+export default ObraAudio

packages/client/src/3d/Phantom.tsx

@@ -1,6 +1,5 @@
 // @ts-nocheck
 import React, { useEffect, useRef } from 'react'
-import { Vector3, Euler } from 'three'
 import Human from './models/Human'
 
 import api, { Transform } from '../store'

packages/client/src/3d/Player.tsx

@@ -144,7 +144,7 @@ const Player = () => {
     const socketEmitTransformInterval = setInterval(() => {
       if (socket && groupRef.current && camera) {
         const cameraRotation = camera.quaternion.toArray()
-        const [x, y, z] = direction.current.toArray()
+        const [x, , z] = direction.current.toArray()
 
         let anim = animations.idle
 

packages/client/src/3d/models/Computer.tsx

@@ -29,7 +29,6 @@ type ComputerProps = {
 
 export default function Model(props: ComputerProps & JSX.IntrinsicElements['group']) {
   const pointerLockControls = useStore((state) => state.pointerLockControls)
-  const setPointerLockStatus = useStore((state) => state.setPointerLockStatus)
   const setModalObra = useStore((state) => state.setModalObra)
 
   const group = useRef<THREE.Group>()
@@ -54,15 +53,17 @@ export default function Model(props: ComputerProps & JSX.IntrinsicElements['grou
       ref={group}
       {...props}
       dispose={null}
-      onPointerOver={(e) => {
+      onPointerOver={() => {
         setCurrentMaterial(true)
       }}
       onPointerOut={() => setCurrentMaterial(false)}
       onClick={() => {
+        if (pointerLockControls) {
           setModalObra(props.obraId)
           setTimeout(() => {
-          pointerLockControls.unlock()
+            pointerLockControls.unlock?.()
           }, 500)
+        }
       }}
     >
       <mesh material={materials.keyboard} geometry={nodes.computer_mesh.geometry} />

packages/client/src/3d/models/Entrance.tsx

@@ -27,7 +27,7 @@ type GLTFResult = GLTF & {
 
 export default function Model(props: JSX.IntrinsicElements['group']) {
   const group = useRef<THREE.Group>()
-  const { scene, materials } = useGLTF('/model/entrance.glb') as GLTFResult
+  const { scene } = useGLTF('/model/entrance.glb') as GLTFResult
   return (
     <group ref={group} {...props} dispose={null}>
       <primitive object={scene} dispose={null} matrixAutoUpdate={false} />

packages/client/src/3d/models/Human.tsx

@@ -14,8 +14,7 @@ import { FrontSide } from 'three/src/constants'
 
 import { HEIGHT } from '../Player'
 
-import api, { State, Transform } from '../../store'
-import { Color } from 'three/src/math/Color'
+import api, { Transform } from '../../store'
 import { MeshNormalMaterial } from 'three/src/materials/MeshNormalMaterial'
 import { Object3D } from 'three/src/core/Object3D'
 import { SkinnedMesh } from 'three/src/objects/SkinnedMesh'

packages/client/src/3d/models/obras/Daniela.tsx

@@ -0,0 +1,14 @@
+import * as THREE from 'three'
+import Proyector from './Proyector'
+import React, { useRef } from 'react'
+import ObraAudio from '../../ObraAudio'
+
+export default function Model(props: JSX.IntrinsicElements['group']) {
+  const group = useRef<THREE.Group>()
+  return (
+    <group ref={group} {...props} dispose={null}>
+      <Proyector videoSrc="/video/daniela.mp4" />
+      <ObraAudio url="/audio/daniela.ogg" loop distance={0.5} position={[1, 1, 1]} />
+    </group>
+  )
+}

packages/client/src/3d/models/obras/DiegoGuido.tsx

@@ -0,0 +1,14 @@
+import * as THREE from 'three'
+import Proyector from './Proyector'
+import React, { useRef } from 'react'
+import ObraAudio from '../../ObraAudio'
+
+export default function Model(props: JSX.IntrinsicElements['group']) {
+  const group = useRef<THREE.Group>()
+  return (
+    <group ref={group} {...props} dispose={null}>
+      <Proyector wide videoSrc="/video/diego_guido.mp4" />
+      <ObraAudio position={[0, 0, -10]} url="/audio/diego_guido.ogg" loop distance={2} />
+    </group>
+  )
+}

packages/client/src/3d/models/obras/Gaston.tsx

@@ -5,6 +5,7 @@ Auto-generated by: https://github.com/pmndrs/gltfjsx
 import * as THREE from 'three'
 import React, { useRef } from 'react'
 import { useGLTF } from '@react-three/drei/useGLTF'
+import ObraAudio from '../../ObraAudio'
 
 import { GLTF } from 'three/examples/jsm/loaders/GLTFLoader'
 
@@ -23,6 +24,13 @@ export default function Model(props: JSX.IntrinsicElements['group']) {
   return (
     <group ref={group} {...props} dispose={null} scale={[0.99, 0.99, 0.99]}>
       <mesh material={materials.gaston} geometry={nodes.logs002.geometry} />
+      <ObraAudio position={[0, 0, 0]} url="/audio/gaston_base.ogg" loop distance={1} />
+      <ObraAudio
+        position={[0, 0, -7]}
+        url="/audio/gaston_movimiento.ogg"
+        loop
+        distance={0.2}
+      />
     </group>
   )
 }

packages/client/src/3d/models/obras/Pedro.tsx

@@ -0,0 +1,14 @@
+import * as THREE from 'three'
+import Proyector from './Proyector'
+import React, { useRef } from 'react'
+import ObraAudio from '../../ObraAudio'
+
+export default function Model(props: JSX.IntrinsicElements['group']) {
+  const group = useRef<THREE.Group>()
+  return (
+    <group ref={group} {...props} dispose={null}>
+      <Proyector wide videoSrc="/video/sagasti.mp4" />
+      <ObraAudio url="/audio/sagasti.ogg" loop distance={2} />
+    </group>
+  )
+}

packages/client/src/3d/models/obras/Proyector.tsx

@@ -3,40 +3,89 @@ Auto-generated by: https://github.com/pmndrs/gltfjsx
 */
 
 import * as THREE from 'three'
-import React, { useRef } from 'react'
+import React, { useRef, useEffect } from 'react'
 import { useGLTF } from '@react-three/drei/useGLTF'
-
+import {
+  Mesh,
+  Color,
+  MeshBasicMaterial,
+  MeshStandardMaterial,
+  VideoTexture,
+  DoubleSide,
+} from 'three'
 import { GLTF } from 'three/examples/jsm/loaders/GLTFLoader'
 
 type GLTFResult = GLTF & {
   nodes: {
-    Cube013: THREE.Mesh
-    ['Cube.013_1']: THREE.Mesh
-    ['Cube.013_2']: THREE.Mesh
+    Cube013: Mesh
+    ['Cube.013_1']: Mesh
+    ['Cube.013_2']: Mesh
+    Cube001: THREE.Mesh
+    ['Cube.001_1']: THREE.Mesh
+    ['Cube.001_2']: THREE.Mesh
   }
   materials: {
-    ['VrHeadset cuerpo.002']: THREE.MeshStandardMaterial
-    ['light.001']: THREE.MeshStandardMaterial
-    ['projection.001']: THREE.MeshStandardMaterial
+    plastic: MeshStandardMaterial
+    projection_light: MeshStandardMaterial
+    projection: MeshStandardMaterial
   }
 }
 
-export default function Model(props: JSX.IntrinsicElements['group']) {
+export default function Model(
+  props: JSX.IntrinsicElements['group'] & { videoSrc: string; wide?: boolean },
+) {
+  const { nodes, materials } = useGLTF(
+    `/model/obras/proyector${props.wide ? '_wide' : ''}.glb`,
+  ) as GLTFResult
   const group = useRef<THREE.Group>()
-  const { nodes, materials } = useGLTF('/model/obras/proyector.glb') as GLTFResult
+  const videoRef = useRef<HTMLVideoElement>(document.createElement('video'))
+  const projectionMaterial = useRef<MeshBasicMaterial>(
+    new MeshBasicMaterial({ color: new Color('white') }),
+  )
+  const videoTexture = useRef<VideoTexture>()
+
+  useEffect(() => {
+    if (videoRef.current && projectionMaterial) {
+      videoRef.current.src = props.videoSrc
+      videoRef.current.load()
+      videoRef.current.loop = true
+      videoRef.current.muted = true
+      videoRef.current.autoplay = true
+      videoRef.current.play()
+
+      videoTexture.current = new VideoTexture(videoRef.current)
+      projectionMaterial.current.map = videoTexture.current
+      projectionMaterial.current.needsUpdate = true
+    }
+  }, [videoRef.current, projectionMaterial.current])
+
   return (
     <group ref={group} {...props} dispose={null}>
-      <group position={[-55.28, 2.67, 85.99]} rotation={[0, Math.PI / 2, 0]}>
+      {!props.wide ? (
+        <>
+          <mesh material={materials.plastic} geometry={nodes.Cube013.geometry} />
           <mesh
-          material={materials['VrHeadset cuerpo.002']}
-          geometry={nodes.Cube013.geometry}
+            material={materials.projection_light}
+            geometry={nodes['Cube.013_1'].geometry}
           />
-        <mesh material={materials['light.001']} geometry={nodes['Cube.013_1'].geometry} />
           <mesh
-          material={materials['projection.001']}
+            material={projectionMaterial.current}
             geometry={nodes['Cube.013_2'].geometry}
           />
-      </group>
+        </>
+      ) : (
+        <>
+          <mesh material={materials.plastic} geometry={nodes.Cube001.geometry} />
+          <mesh
+            material={materials.projection_light}
+            geometry={nodes['Cube.001_1'].geometry}
+          />
+          <mesh
+            material={projectionMaterial.current}
+            geometry={nodes['Cube.001_2'].geometry}
+          />
+        </>
+      )}
     </group>
   )
 }

packages/client/src/3d/models/obras/Proyector_wide.tsx

@@ -0,0 +1,36 @@
+/*
+Auto-generated by: https://github.com/pmndrs/gltfjsx
+*/
+
+import * as THREE from 'three'
+import React, { useRef } from 'react'
+import { useGLTF } from '@react-three/drei/useGLTF'
+
+import { GLTF } from 'three/examples/jsm/loaders/GLTFLoader'
+
+type GLTFResult = GLTF & {
+  nodes: {
+    Cube001: THREE.Mesh
+    ['Cube.001_1']: THREE.Mesh
+    ['Cube.001_2']: THREE.Mesh
+  }
+  materials: {
+    plastic: THREE.MeshStandardMaterial
+    projection_light: THREE.MeshStandardMaterial
+    projection: THREE.MeshStandardMaterial
+  }
+}
+
+export default function Model(props: JSX.IntrinsicElements['group']) {
+  const group = useRef<THREE.Group>()
+  const { nodes, materials } = useGLTF('/proyector_wide.glb') as GLTFResult
+  return (
+    <group ref={group} {...props} dispose={null}>
+      <mesh material={materials.plastic} geometry={nodes.Cube001.geometry} />
+      <mesh material={materials.projection_light} geometry={nodes['Cube.001_1'].geometry} />
+      <mesh material={materials.projection} geometry={nodes['Cube.001_2'].geometry} />
+    </group>
+  )
+}
+
+useGLTF.preload('/proyector_wide.glb')

packages/client/src/components/MenuOverlay.tsx

@@ -46,7 +46,6 @@ const MenuOverlay: React.FC = () => {
       pointerLockControls.lock?.()
     }
   }
-  console.log(modalObra)
 
   return (
     <>