Interaction system

overview the interaction system enables entities in the ir engine to respond to user actions and interact with each other it provides a framework for creating interactive objects that can be clicked, grabbed, mounted, or otherwise manipulated by avatars or other entities by defining how objects respond to proximity, input, and other triggers, the system creates a responsive and engaging virtual environment this chapter explores the concepts, structure, and implementation of the interaction system within the ir engine core components interactable component the interactablecomponent is the foundation for making objects responsive to user actions // simplified from src/interaction/components/interactablecomponent ts import { definecomponent, s } from '@ir engine/ecs'; export const interactablecomponent = definecomponent({ name 'interactablecomponent', schema s object({ label s string(), // text to display (e g , "press e") activationdistance s number({ default 2 }), // how close the avatar needs to be clickinteract s bool({ default true }), // can be activated by clicking/key press callbacks s array(s object({ callbackid s string(), // id of the callback function target s entity() // entity where the callback is defined })) }) }); this component marks an entity as interactive defines a label to display when an avatar is nearby specifies the distance at which interaction becomes available determines whether the object can be activated by clicking or key press lists callback functions to execute when interaction occurs grabbable component the grabbablecomponent enables objects to be picked up and held // simplified from src/grabbable/grabbablecomponent ts import { definecomponent, s } from '@ir engine/ecs'; export const grabbablecomponent = definecomponent({ name 'grabbablecomponent', schema s object({ grabdistance s number({ default 1 }), // maximum distance for grabbing attachmentoffset s object({ // position offset when grabbed x s number({ default 0 }), y s number({ default 0 }), z s number({ default 0 }) }), throwvelocitymultiplier s number({ default 1 }) // affects throw strength }) }); // added when an object is currently being grabbed export const grabbedcomponent = definecomponent({ name 'grabbedcomponent', schema s object({ grabberentity s entity(), // entity doing the grabbing attachmentpoint s string() // e g , "lefthand" or "righthand" }) }); these components mark an entity as something that can be picked up define how the object behaves when grabbed track which entity is currently holding the object specify where the object attaches to the grabber mount point component the mountpointcomponent creates attachment points for avatars or other entities // simplified from src/scene/components/mountpointcomponent ts import { definecomponent, s } from '@ir engine/ecs'; export const mountpointcomponent = definecomponent({ name 'mountpointcomponent', schema s object({ type s string(), // e g , "seat", "driver", "passenger" offset s object({ // position offset for the mounted entity x s number({ default 0 }), y s number({ default 0 }), z s number({ default 0 }) }), rotation s object({ // rotation for the mounted entity x s number({ default 0 }), y s number({ default 0 }), z s number({ default 0 }) }), animation s string({ default "" }) // animation to play when mounted }) }); // added to an entity that is currently mounted export const sittingcomponent = definecomponent({ name 'sittingcomponent', schema s object({ mountpointentity s entity() // entity with the mountpointcomponent }) }); these components define locations where entities can attach (like seats in a vehicle) specify the position and orientation for mounted entities indicate which animation to play when mounted track which entities are currently mounted callback component the callbackcomponent defines functions that execute when interactions occur // simplified from src/spatial/src/common/callbackcomponent ts import { definecomponent, s } from '@ir engine/ecs'; export const callbackcomponent = definecomponent({ name 'callbackcomponent', schema s object({ callbacks s record(s string(), s function()) }) }); // helper function to set a callback export function setcallback(entity, callbackid, callbackfunction) { const callbacks = getcomponent(entity, callbackcomponent)? callbacks || {}; callbacks\[callbackid] = callbackfunction; setcomponent(entity, callbackcomponent, { callbacks }); } this component stores callback functions by id allows entities to define custom behavior for interactions provides a way to register, update, and execute callbacks interaction systems several systems work together to process these components and enable interactions interactable system the interactablesystem manages proximity detection and interaction triggering // simplified concept from src/interaction/systems/interactablesystem ts const interactablesystem = definesystem({ uuid 'ir engine interactablesystem', execute () => { // find all entities with interactablecomponent const interactables = interactablequery(); // find the local avatar entity const avatarentity = getlocalavatarentity(); if (!avatarentity) return; // get the avatar's position const avatarposition = getcomponent(avatarentity, transformcomponent) position; // process each interactable for (const entity of interactables) { const interactable = getcomponent(entity, interactablecomponent); const transform = getcomponent(entity, transformcomponent); // calculate distance to avatar const distance = calculatedistance(avatarposition, transform position); // check if within activation distance if (distance <= interactable activationdistance) { // show interaction label showinteractionlabel(entity, interactable label); // check for interaction input if (interactable clickinteract && isinteractkeypressed()) { // execute callbacks executeinteractioncallbacks(avatarentity, entity, interactable callbacks); } } else { // hide interaction label hideinteractionlabel(entity); } } } }); this system processes all entities with interactablecomponent calculates distances between the avatar and interactable objects shows or hides interaction labels based on proximity detects interaction input (like key presses) executes callback functions when interaction occurs grabbable system the grabbablesystem handles object grabbing and manipulation // simplified concept from src/grabbable/grabbablesystem tsx const grabbablesystem = definesystem({ uuid 'ir engine grabbablesystem', execute () => { // find all entities that are currently grabbed const grabbedentities = grabbedquery(); for (const entity of grabbedentities) { const grabbed = getcomponent(entity, grabbedcomponent); const grabber = grabbed grabberentity; // get the attachment point (hand) transform const attachmentpoint = getattachmentpointtransform(grabber, grabbed attachmentpoint); if (!attachmentpoint) continue; // update the grabbed object's transform to follow the hand const transform = getmutablecomponent(entity, transformcomponent); transform position = calculategrabbedposition(attachmentpoint, entity); transform rotation = calculategrabbedrotation(attachmentpoint, entity); // if the object has physics, update its kinematic target if (hascomponent(entity, rigidbodycomponent)) { const rigidbody = getmutablecomponent(entity, rigidbodycomponent); rigidbody targetkinematicposition = transform position; rigidbody targetkinematicrotation = transform rotation; } } // check for grab/release input if (isgrabinputpressed()) { const avatarentity = getlocalavatarentity(); const nearbygrabbable = findnearestgrabbable(avatarentity); if (nearbygrabbable) { // grab the object grabbablecomponent grab(avatarentity, nearbygrabbable); } } if (isreleaseinputpressed()) { const avatarentity = getlocalavatarentity(); const heldobject = getheldobject(avatarentity); if (heldobject) { // release the object grabbablecomponent drop(avatarentity, heldobject); } } } }); this system updates the position and rotation of grabbed objects to follow the grabber's hand handles physics interactions for grabbed objects detects grab and release input initiates grab and drop actions mount point system the mountpointsystem manages entity mounting and dismounting // simplified concept from src/interaction/systems/mountpointsystem ts const mountpointsystem = definesystem({ uuid 'ir engine mountpointsystem', execute () => { // find all entities that are currently mounted/sitting const sittingentities = sittingquery(); for (const entity of sittingentities) { const sitting = getcomponent(entity, sittingcomponent); const mountpoint = getcomponent(sitting mountpointentity, mountpointcomponent); // get the mount point's transform const mounttransform = getcomponent(sitting mountpointentity, transformcomponent); // update the mounted entity's transform const transform = getmutablecomponent(entity, transformcomponent); transform position = calculatemountedposition(mounttransform, mountpoint); transform rotation = calculatemountedrotation(mounttransform, mountpoint); // check for dismount input if (isdismountinputpressed(entity)) { // unmount the entity mountpointcomponent unmountentity(entity); } } } }); this system updates the position and rotation of mounted entities aligns entities with their mount points detects dismount input initiates unmounting actions interaction workflow the process of interaction follows this general workflow sequencediagram participant player participant avatar participant inputsystem as input system participant interactablesys as interactablesystem participant buttonentity as button entity participant callbacklogic as callback function player >>avatar moves near button interactablesys >>buttonentity detects proximity interactablesys >>player shows "press e" label player >>inputsystem presses 'e' key inputsystem >>interactablesys notifies of key press interactablesys >>buttonentity checks for callbacks buttonentity >>callbacklogic executes callback callbacklogic >>player performs action (e g , shows message) this diagram illustrates the proximity detection that triggers ui feedback the input detection that initiates interaction the callback execution that performs the actual action the feedback loop that informs the player of the result practical example let's create a simple interactive button that displays a message when pressed // create an entity for the button const buttonentity = createentity(); // add visual components setcomponent(buttonentity, transformcomponent, { position { x 0, y 1, z 2 }, rotation { x 0, y 0, z 0 }, scale { x 0 2, y 0 2, z 0 05 } }); setcomponent(buttonentity, meshcomponent, { geometrytype 'box', materialprops { color 'red' } }); // make the button interactable setcomponent(buttonentity, interactablecomponent, { label "press e to activate", activationdistance 2, clickinteract true, callbacks \[ { callbackid "showmessagecallback", target buttonentity } ] }); // define the callback function setcallback(buttonentity, "showmessagecallback", (interactingentity, targetentity) => { console log("button activated! you pressed it!"); // in a real application, this could // open a door // play a sound // change a game state // trigger an animation }); this example demonstrates creating a button entity with visual representation making it interactable with a label and activation distance defining a callback function that executes when the button is pressed additional interaction examples grabbable object creating an object that can be picked up and held // create an entity for a grabbable box const boxentity = createentity(); // add visual components setcomponent(boxentity, transformcomponent, { position { x 1, y 1, z 2 }, rotation { x 0, y 0, z 0 }, scale { x 0 2, y 0 2, z 0 2 } }); setcomponent(boxentity, meshcomponent, { geometrytype 'box', materialprops { color 'blue' } }); // add physics for realistic behavior setcomponent(boxentity, rigidbodycomponent, { type 'dynamic', mass 1 }); // make the box grabbable setcomponent(boxentity, grabbablecomponent, { grabdistance 1 5, throwvelocitymultiplier 1 5 }); // optional add a callback for when the box is grabbed setcallback(boxentity, grabbablecomponent grabbablecallbackname, (grabber, grabbeditem) => { console log("box grabbed by ", grabber); // could trigger a sound, particle effect, etc }); this example shows creating a box entity with visual and physics components making it grabbable with specific grab distance and throw properties adding a callback that executes when the box is grabbed mountable chair creating a chair that an avatar can sit on // create an entity for a chair const chairentity = createentity(); // add visual components setcomponent(chairentity, transformcomponent, { position { x 1, y 0, z 2 }, rotation { x 0, y 0, z 0 }, scale { x 1, y 1, z 1 } }); setcomponent(chairentity, gltfcomponent, { src "models/chair glb" }); // add a mount point for sitting setcomponent(chairentity, mountpointcomponent, { type "seat", offset { x 0, y 0 5, z 0 }, rotation { x 0, y 0, z 0 }, animation "sitting" }); // make the chair interactable to initiate sitting setcomponent(chairentity, interactablecomponent, { label "sit", activationdistance 2, clickinteract true, callbacks \[ { callbackid mountpointcomponent mountcallbackname, target chairentity } ] }); this example demonstrates creating a chair entity with a 3d model adding a mount point that defines where and how an avatar sits making the chair interactable to trigger the mounting action implementation details interactable detection the interactablesystem uses several methods to detect potential interactions // simplified concept from src/interaction/functions/interactablefunctions ts function gatheravailableinteractables(interactables) { const result = \[]; const avatarentity = getlocalavatarentity(); if (!avatarentity) return result; const avatarposition = getcomponent(avatarentity, transformcomponent) position; // process each interactable for (const entity of interactables) { const interactable = getcomponent(entity, interactablecomponent); const transform = getcomponent(entity, transformcomponent); // calculate distance to avatar const distance = calculatedistance(avatarposition, transform position); // check if within activation distance if (distance <= interactable activationdistance) { result push({ entity, distance, interactable }); } } // sort by distance (closest first) result sort((a, b) => a distance b distance); return result; } this function collects all interactable entities within activation distance of the avatar calculates the distance to each interactable sorts them by distance to prioritize closer interactions returns a list of potential interactions callback execution when an interaction occurs, the system executes the appropriate callbacks // simplified concept function executeinteractioncallbacks(interactingentity, targetentity, callbacks) { for (const callback of callbacks) { const { callbackid, target } = callback; // get the callback component from the target entity const callbackcomponent = getcomponent(target, callbackcomponent); if (!callbackcomponent) continue; // get the callback function const callbackfunction = callbackcomponent callbacks\[callbackid]; if (!callbackfunction) continue; // execute the callback callbackfunction(interactingentity, targetentity); } } this function processes each callback defined in the interaction retrieves the callback function from the target entity executes the function with the interacting and target entities as parameters grab mechanics the grabbablecomponent provides static methods for grabbing and dropping objects // simplified from src/grabbable/grabbablecomponent ts grabbablecomponent grab = (grabberentity, grabbableentity, handedness) => { // validate entities have the required components if (!hascomponent(grabberentity, grabbercomponent) || !hascomponent(grabbableentity, grabbablecomponent)) { return; } // dispatch a network action to ensure consistency across clients dispatchaction( grabbablenetworkaction setgrabbedobject({ entityuuid getentityuuid(grabbableentity), grabberentityuuid getentityuuid(grabberentity), grabbed true, attachmentpoint handedness // 'left' or 'right' hand }) ); }; grabbablecomponent drop = (grabberentity, grabbableentity) => { // similar validation // dispatch a network action to release the object dispatchaction( grabbablenetworkaction setgrabbedobject({ entityuuid getentityuuid(grabbableentity), grabberentityuuid getentityuuid(grabberentity), grabbed false }) ); }; these methods validate that the entities have the required components dispatch network actions to ensure consistency across clients trigger the actual grab or drop operation benefits of the interaction system the interaction system provides several key benefits modularity components can be mixed and matched to create different types of interactions flexibility callbacks allow for custom behavior without modifying the core system consistency standard interaction patterns create a predictable user experience extensibility new interaction types can be added by creating new components and systems feedback visual and audio cues help users understand available interactions networking actions can be synchronized across clients for multiplayer experiences these benefits make the interaction system a powerful tool for creating engaging and responsive virtual environments next steps with an understanding of how entities can interact with each other, the next chapter explores how to create complex behaviors and game logic without writing extensive code next visual scripting system docid\ ahmt ieibephl08gjlc8l