Skip to content

Commit

Permalink
Merge pull request #6451 from mozilla/bitecs-snap-object-menu
Browse files Browse the repository at this point in the history
BitECS snap object menu support
  • Loading branch information
keianhzo authored Jan 24, 2024
2 parents add9afa + 75efc61 commit ced6609
Show file tree
Hide file tree
Showing 14 changed files with 224 additions and 26 deletions.
Binary file added src/assets/snap.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
6 changes: 5 additions & 1 deletion src/bit-components.js
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,8 @@ export const MediaContentBounds = defineComponent({
});
export const MediaInfo = defineComponent({
accessibleUrl: Types.ui32,
contentType: Types.ui32
contentType: Types.ui32,
mediaType: Types.ui8
});
MediaInfo.accessibleUrl[$isStringType] = true;
MediaInfo.contentType[$isStringType] = true;
Expand Down Expand Up @@ -279,6 +280,7 @@ export const VideoMenu = defineComponent({
headRef: Types.eid,
playIndicatorRef: Types.eid,
pauseIndicatorRef: Types.eid,
snapRef: Types.eid,
clearTargetTimer: Types.f64
});
export const AudioEmitter = defineComponent({
Expand Down Expand Up @@ -353,12 +355,14 @@ export const PDFMenu = defineComponent({
prevButtonRef: Types.eid,
nextButtonRef: Types.eid,
pageLabelRef: Types.eid,
snapRef: Types.eid,
targetRef: Types.eid,
clearTargetTimer: Types.f64
});
export const ObjectMenuTarget = defineComponent({
flags: Types.ui8
});
export const MediaSnapped = defineComponent();
export const NetworkDebug = defineComponent();
export const NetworkDebugRef = defineComponent({
ref: Types.eid
Expand Down
1 change: 1 addition & 0 deletions src/bit-systems/media-loading.ts
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ function* loadMedia(world: HubsWorld, eid: EntityID) {
addComponent(world, MediaInfo, media);
MediaInfo.accessibleUrl[media] = APP.getSid(urlData.accessibleUrl);
MediaInfo.contentType[media] = APP.getSid(urlData.contentType);
MediaInfo.mediaType[media] = urlData.mediaType || 0;
} catch (e) {
console.error(e);
media = renderAsEntity(world, ErrorObject());
Expand Down
6 changes: 4 additions & 2 deletions src/bit-systems/object-menu-transform-system.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const tmpMat42 = new Matrix4();
const aabb = new Box3();
const sphere = new Sphere();
const yVector = new Vector3(0, 1, 0);
const UNIT_V3 = new Vector3(1, 1, 1);

// Calculate the AABB without accounting for the root object rotation
function getAABB(obj: Object3D, box: Box3, onlyVisible: boolean = false) {
Expand Down Expand Up @@ -83,8 +84,9 @@ function transformMenu(world: HubsWorld, menu: EntityID) {
// For now we are defaulting to the current AFrame behavior.
} else {
targetObj.updateMatrices(true, true);
tmpMat4.copy(targetObj.matrixWorld);
tmpMat4.decompose(tmpVec1, tmpQuat1, tmpVec2);
targetObj.matrixWorld.decompose(tmpVec1, tmpQuat1, tmpVec2);
tmpMat42.compose(tmpVec1, tmpQuat1, UNIT_V3);
tmpMat4.copy(tmpMat42);

const isFacing = isFacingCamera(targetObj);
if (!isFacing) {
Expand Down
12 changes: 5 additions & 7 deletions src/bit-systems/pdf-menu-system.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
MediaLoader,
MediaPDF,
MediaPDFUpdated,
MediaSnapped,
NetworkedPDF,
ObjectMenuTransform,
PDFMenu
Expand All @@ -25,6 +26,7 @@ function setCursorRaycastable(world: HubsWorld, menu: EntityID, enable: boolean)
change(world, CursorRaycastable, menu);
change(world, CursorRaycastable, PDFMenu.prevButtonRef[menu]);
change(world, CursorRaycastable, PDFMenu.nextButtonRef[menu]);
change(world, CursorRaycastable, PDFMenu.snapRef[menu]);
}

function clicked(world: HubsWorld, eid: EntityID) {
Expand Down Expand Up @@ -83,6 +85,9 @@ function handleClicks(world: HubsWorld, menu: EntityID) {
} else if (clicked(world, PDFMenu.prevButtonRef[menu])) {
const pdf = PDFMenu.targetRef[menu];
setPage(world, pdf, MediaPDF.pageNumber[pdf] - 1);
} else if (clicked(world, PDFMenu.snapRef[menu])) {
const pdf = PDFMenu.targetRef[menu];
addComponent(world, MediaSnapped, pdf);
}
}

Expand All @@ -107,13 +112,6 @@ function flushToObject3Ds(world: HubsWorld, menu: EntityID, frozen: boolean) {
ObjectMenuTransform.flags[menu] &= ~ObjectMenuTransformFlags.Enabled;
}

[PDFMenu.prevButtonRef[menu], PDFMenu.nextButtonRef[menu]].forEach(buttonRef => {
const buttonObj = world.eid2obj.get(buttonRef)!;
// Parent visibility doesn't block raycasting, so we must set each button to be invisible
// TODO: Ensure that children of invisible entities aren't raycastable
buttonObj.visible = visible;
});

if (target) {
const numPages = PDFResourcesMap.get(target)!.pdf.numPages;
(world.eid2obj.get(PDFMenu.pageLabelRef[menu]) as Text).text = `${MediaPDF.pageNumber[target]} / ${numPages}`;
Expand Down
127 changes: 127 additions & 0 deletions src/bit-systems/snap-media-system.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import { defineQuery, enterQuery, entityExists, exitQuery, hasComponent, removeComponent } from "bitecs";
import { HubsWorld } from "../app";
import { MediaPDF, MediaSnapped, MediaVideo, MediaVideoData } from "../bit-components";
import { SOUND_CAMERA_TOOL_TOOK_SNAPSHOT } from "../systems/sound-effects-system";
import { JobRunner } from "../utils/coroutine-utils";
import { EntityID } from "../utils/networking-types";
import { PDFResourcesMap } from "./pdf-system";
import { guessContentType } from "../utils/media-url-utils";
import { upload } from "../utils/media-utils";
import qsTruthy from "../utils/qs_truthy";
import { createNetworkedMedia } from "../utils/create-networked-entity";
import { Quaternion, Vector3 } from "three";
import { MediaLoaderParams } from "../inflators/media-loader";
import { animate } from "../utils/animate";
import { easeOutQuadratic } from "../utils/easing";
import { crNextFrame } from "../utils/coroutine";

const TYPE_IMG_PNG = { type: "image/png" };

const finalPos = new Vector3();
const intialPos = new Vector3();
const tmpQuat = new Quaternion();
export function* snapMedia(world: HubsWorld, eid: EntityID) {
let canvas: HTMLCanvasElement | undefined;
if (hasComponent(world, MediaPDF, eid)) {
const res = PDFResourcesMap.get(eid);
canvas = res?.canvas;
} else if (hasComponent(world, MediaVideo, eid)) {
const video = MediaVideoData.get(eid)!;
if (video) {
canvas = document.createElement("canvas");
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
canvas.getContext("2d")!.drawImage(video, 0, 0, canvas.width, canvas.height);
}
}

if (canvas) {
const blob = new Promise((resolve, reject) => {
if (canvas) {
canvas.toBlob(resolve);
} else {
reject();
}
});
if (blob) {
const file = new File([yield blob], "snap.png", TYPE_IMG_PNG);
const desiredContentType = file.type || guessContentType(file.name);
const uploadPromise = new Promise((resolve, reject) => {
upload(file, desiredContentType)
.then(function (response) {
const srcUrl = new URL(response.origin);
srcUrl.searchParams.set("token", response.meta.access_token);
resolve({
src: srcUrl.href,
recenter: true,
resize: !qsTruthy("noResize"),
animateLoad: true,
fileId: response.file_id,
isObjectMenuTarget: true
});
})
.catch(e => {
console.error("Media upload failed", e);
reject({
src: "error",
recenter: true,
resize: !qsTruthy("noResize"),
animateLoad: true,
isObjectMenuTarget: true
});
});
});

const params: MediaLoaderParams = yield uploadPromise;
const snappedEid = createNetworkedMedia(APP.world, params);
const idx = (Math.floor(Math.random() * 100) % 6) + 3;
finalPos.set(
Math.cos(Math.PI * 2 * (idx / 6.0)) * 0.75,
Math.sin(Math.PI * 2 * (idx / 6.0)) * 0.75,
-0.05 + idx * 0.001
);
const sourceObj = world.eid2obj.get(eid)!;
sourceObj.localToWorld(finalPos);
const snappedObj = APP.world.eid2obj.get(snappedEid)!;
sourceObj.getWorldQuaternion(tmpQuat);
snappedObj.quaternion.copy(tmpQuat);

const onAnimate = ([pos]: [Vector3]) => {
snappedObj.position.copy(pos);
snappedObj.matrixNeedsUpdate = true;
};
yield crNextFrame();
sourceObj.getWorldPosition(intialPos);
yield* animate({
properties: [[intialPos, finalPos]],
durationMS: 400,
easing: easeOutQuadratic,
fn: onAnimate
});
} else {
console.error("Snapped image creation error");
}
}

if (entityExists(world, eid)) {
removeComponent(world, MediaSnapped, eid);
}
}

const jobs = new JobRunner();
const snappedMediaQuery = defineQuery([MediaSnapped]);
const snappedEnterQuery = enterQuery(snappedMediaQuery);
const snappedExitQuery = exitQuery(snappedMediaQuery);
export function snapMediaSystem(world: HubsWorld, sfxSystem: any) {
snappedExitQuery(world).forEach(eid => {
jobs.stop(eid);
});
snappedEnterQuery(world).forEach(eid => {
sfxSystem.playSoundOneShot(SOUND_CAMERA_TOOL_TOOK_SNAPSHOT);

if (!jobs.has(eid)) {
jobs.add(eid, () => snapMedia(world, eid));
}
});
jobs.tick();
}
10 changes: 9 additions & 1 deletion src/bit-systems/video-menu-system.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ import {
HeldRemoteRight,
HoveredRemoteRight,
Interacted,
MediaInfo,
MediaLoader,
MediaSnapped,
MediaVideo,
MediaVideoData,
MediaVideoUpdated,
Expand All @@ -27,6 +29,7 @@ import { Emitter2Audio } from "./audio-emitter-system";
import { EntityID } from "../utils/networking-types";
import { findAncestorWithComponent, hasAnyComponent } from "../utils/bit-utils";
import { ObjectMenuTransformFlags } from "../inflators/object-menu-transform";
import { MediaType } from "../utils/media-utils";

const videoMenuQuery = defineQuery([VideoMenu]);
const hoveredQuery = defineQuery([HoveredRemoteRight]);
Expand All @@ -38,6 +41,7 @@ function setCursorRaycastable(world: HubsWorld, menu: number, enable: boolean) {
change(world, CursorRaycastable, VideoMenu.trackRef[menu]);
change(world, CursorRaycastable, VideoMenu.playIndicatorRef[menu]);
change(world, CursorRaycastable, VideoMenu.pauseIndicatorRef[menu]);
change(world, CursorRaycastable, VideoMenu.snapRef[menu]);
}

const intersectInThePlaneOf = (() => {
Expand Down Expand Up @@ -110,10 +114,11 @@ function flushToObject3Ds(world: HubsWorld, menu: EntityID, frozen: boolean) {
APP.world.scene.add(obj);
ObjectMenuTransform.targetObjectRef[menu] = target;
ObjectMenuTransform.flags[menu] |= ObjectMenuTransformFlags.Enabled;
const snapButton = world.eid2obj.get(VideoMenu.snapRef[menu])!;
snapButton.visible = MediaInfo.mediaType[target] === MediaType.VIDEO;
} else {
obj.removeFromParent();
setCursorRaycastable(world, menu, false);

ObjectMenuTransform.flags[menu] &= ~ObjectMenuTransformFlags.Enabled;
}
}
Expand Down Expand Up @@ -142,6 +147,9 @@ function handleClicks(world: HubsWorld, menu: EntityID) {
addComponent(world, EntityStateDirty, videoEid);
}
addComponent(world, MediaVideoUpdated, videoEid);
} else if (clicked(world, VideoMenu.snapRef[menu])) {
const video = VideoMenu.videoRef[menu];
addComponent(world, MediaSnapped, video);
}
}

Expand Down
3 changes: 2 additions & 1 deletion src/hub.js
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ import { renderAsEntity } from "./utils/jsx-entity";
import { VideoMenuPrefab, loadVideoMenuButtonIcons } from "./prefabs/video-menu";
import { loadObjectMenuButtonIcons, ObjectMenuPrefab } from "./prefabs/object-menu";
import { loadMirrorMenuButtonIcons, MirrorMenuPrefab } from "./prefabs/mirror-menu";
import { loadPDFMenuButtonIcons } from "./prefabs/pdf-menu";
import { LinkHoverMenuPrefab } from "./prefabs/link-hover-menu";
import { PDFMenuPrefab } from "./prefabs/pdf-menu";
import { loadWaypointPreviewModel, WaypointPreview } from "./prefabs/waypoint-preview";
Expand All @@ -206,7 +207,7 @@ function addToScene(entityDef, visible) {
obj.visible = !!visible;
});
}
preload(addToScene(PDFMenuPrefab(), false));
preload(loadPDFMenuButtonIcons().then(() => addToScene(PDFMenuPrefab(), false)));
preload(loadObjectMenuButtonIcons().then(() => addToScene(ObjectMenuPrefab(), false)));
preload(loadMirrorMenuButtonIcons().then(() => addToScene(MirrorMenuPrefab(), false)));
preload(addToScene(LinkHoverMenuPrefab(), false));
Expand Down
1 change: 1 addition & 0 deletions src/inflators/pdf.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { createPlaneBufferGeometry } from "../utils/three-utils";
export interface PDFResources {
pdf: PDFDocumentProxy;
material: MeshBasicMaterial;
canvas: HTMLCanvasElement;
canvasContext: CanvasRenderingContext2D;
}

Expand Down
40 changes: 33 additions & 7 deletions src/prefabs/pdf-menu.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,17 @@ import { Color } from "three";
import { ArrayVec3, Attrs, createElementEntity, createRef } from "../utils/jsx-entity";
import { Button3D, BUTTON_TYPES } from "./button3D";
import { Label } from "./camera-tool";
import { loadTexture, loadTextureFromCache } from "../utils/load-texture";
import snapIconSrc from "../assets/spawn_message.png";

const BUTTON_HEIGHT = 0.2;
const BUTTON_SCALE: ArrayVec3 = [0.4, 0.4, 0.4];
const BUTTON_WIDTH = 0.3;
const BUTTON_SCALE: ArrayVec3 = [0.6, 0.6, 0.6];
const BIG_BUTTON_SCALE: ArrayVec3 = [0.8, 0.8, 0.8];
const BUTTON_WIDTH = 0.2;

export async function loadPDFMenuButtonIcons() {
return Promise.all([loadTexture(snapIconSrc, 1, "image/png")]);
}

interface PDFPageButtonProps extends Attrs {
text: string;
Expand All @@ -16,7 +23,7 @@ function PDFPageButton(props: PDFPageButtonProps) {
return (
<Button3D
name={props.name}
scale={BUTTON_SCALE}
scale={BIG_BUTTON_SCALE}
width={BUTTON_WIDTH}
height={BUTTON_HEIGHT}
type={BUTTON_TYPES.ACTION}
Expand All @@ -25,25 +32,44 @@ function PDFPageButton(props: PDFPageButtonProps) {
);
}

function SnapButton(props: Attrs) {
const { texture, cacheKey } = loadTextureFromCache(snapIconSrc, 1);
return (
<Button3D
name="Remove Button"
scale={BUTTON_SCALE}
width={BUTTON_HEIGHT}
height={BUTTON_WIDTH}
type={BUTTON_TYPES.ACTION}
icon={{ texture, cacheKey, scale: [0.165, 0.165, 0.165] }}
{...props}
/>
);
}

const UI_Z = 0.001;
const POSITION_PREV: ArrayVec3 = [-0.45, 0.0, UI_Z];
const POSITION_NEXT: ArrayVec3 = [0.45, 0.0, UI_Z];
const POSITION_LABEL: ArrayVec3 = [0.0, -0.35, UI_Z];
const POSITION_PREV: ArrayVec3 = [-0.35, 0.0, UI_Z];
const POSITION_NEXT: ArrayVec3 = [0.35, 0.0, UI_Z];
const POSITION_LABEL: ArrayVec3 = [0.0, -0.45, UI_Z];
const POSITION_SNAP: ArrayVec3 = [0.0, 0.45, UI_Z];
const PAGE_LABEL_COLOR = new Color(0.1, 0.1, 0.1);
export function PDFMenuPrefab() {
const refPrev = createRef();
const refNext = createRef();
const refLabel = createRef();
const refSnap = createRef();
return (
<entity
name="PDF Menu"
objectMenuTransform={{ center: false }}
pdfMenu={{
prevButtonRef: refPrev,
nextButtonRef: refNext,
pageLabelRef: refLabel
pageLabelRef: refLabel,
snapRef: refSnap
}}
>
<SnapButton name="Snap Button" ref={refSnap} position={POSITION_SNAP} />
<PDFPageButton name="Previous Page Button" text="<" ref={refPrev} position={POSITION_PREV} />
<PDFPageButton name="Next Page Button" text=">" ref={refNext} position={POSITION_NEXT} />
<Label ref={refLabel} position={POSITION_LABEL} text={{ color: PAGE_LABEL_COLOR }} />
Expand Down
Loading

0 comments on commit ced6609

Please sign in to comment.