Merge pull request #6111 from mozilla/bitecs-media-video-params

Add support for media video and image parameters

src/bit-components.js

@@ -167,9 +167,16 @@ export const SceneLoader = defineComponent({ src: Types.ui32 });
 SceneLoader.src[$isStringType] = true;
 
 export const MediaImage = defineComponent({
-  cacheKey: Types.ui32
+  cacheKey: Types.ui32,
+  projection: Types.ui8,
+  alphaMode: Types.ui8,
+  alphaCutoff: Types.f32
 });
 MediaImage.cacheKey[$isStringType] = true;
+/**
+ * @type {Map<EntityId, ImageLoaderParams}>}
+ */
+export const MediaImageLoaderData = new Map();
 
 export const NetworkedPDF = defineComponent({
   pageNumber: Types.ui8
@@ -180,9 +187,14 @@ export const MediaPDF = defineComponent({
 MediaPDF.map = new Map();
 
 export const MediaVideo = defineComponent({
-  autoPlay: Types.ui8,
-  ratio: Types.f32
+  ratio: Types.f32,
+  flags: Types.ui8,
+  projection: Types.ui8
 });
+/**
+ * @type {Map<EntityId, VideoLoaderParams}>}
+ */
+export const MediaVideoLoaderData = new Map();
 /**
  * @type {Map<EntityId, HTMLVideoElement}>}
  */

src/bit-systems/media-loading.ts

@@ -69,21 +69,28 @@ export function* waitForMediaLoaded(world: HubsWorld, eid: EntityID) {
 const loaderForMediaType = {
   [MediaType.IMAGE]: (
     world: HubsWorld,
-    { accessibleUrl, contentType }: { accessibleUrl: string; contentType: string }
-  ) => loadImage(world, accessibleUrl, contentType),
+    eid: EntityID,
+    { accessibleUrl, contentType }: { accessibleUrl: string, contentType: string }
+  ) => loadImage(world, eid, accessibleUrl, contentType),
   [MediaType.VIDEO]: (
     world: HubsWorld,
-    { accessibleUrl, contentType }: { accessibleUrl: string; contentType: string }
-  ) => loadVideo(world, accessibleUrl, contentType),
+    eid: EntityID,
+    { accessibleUrl, contentType }: { accessibleUrl: string, contentType: string }
+  ) => loadVideo(world, eid, accessibleUrl, contentType),
   [MediaType.MODEL]: (
     world: HubsWorld,
-    { accessibleUrl, contentType }: { accessibleUrl: string; contentType: string }
+    eid: EntityID,
+    { accessibleUrl, contentType }: { accessibleUrl: string, contentType: string }
   ) => loadModel(world, accessibleUrl, contentType, true),
-  [MediaType.PDF]: (world: HubsWorld, { accessibleUrl }: { accessibleUrl: string }) => loadPDF(world, accessibleUrl),
-  [MediaType.AUDIO]: (world: HubsWorld, { accessibleUrl }: { accessibleUrl: string }) =>
-    loadAudio(world, accessibleUrl),
-  [MediaType.HTML]: (world: HubsWorld, { canonicalUrl, thumbnail }: { canonicalUrl: string; thumbnail: string }) =>
-    loadHtml(world, canonicalUrl, thumbnail)
+  [MediaType.PDF]: (world: HubsWorld, eid: EntityID, { accessibleUrl }: { accessibleUrl: string }) =>
+    loadPDF(world, accessibleUrl),
+  [MediaType.AUDIO]: (world: HubsWorld, eid: EntityID, { accessibleUrl }: { accessibleUrl: string }) =>
+    loadAudio(world, eid, accessibleUrl),
+  [MediaType.HTML]: (
+    world: HubsWorld,
+    eid: EntityID,
+    { canonicalUrl, thumbnail }: { canonicalUrl: string, thumbnail: string }
+  ) => loadHtml(world, canonicalUrl, thumbnail)
 };
 
 export const MEDIA_LOADER_FLAGS = {
@@ -197,7 +204,7 @@ function* loadMedia(world: HubsWorld, eid: EntityID) {
     if (!loader) {
       throw new UnsupportedMediaTypeError(eid, urlData.mediaType);
     }
-    media = yield* loader(world, urlData);
+    media = yield* loader(world, eid, urlData);
     addComponent(world, MediaLoaded, media);
   } catch (e) {
     console.error(e);

src/bit-systems/video-menu-system.ts

@@ -23,6 +23,7 @@ import { coroutine } from "../utils/coroutine";
 import { easeOutQuadratic } from "../utils/easing";
 import { isFacingCamera } from "../utils/three-utils";
 import { Emitter2Audio } from "./audio-emitter-system";
+import { VIDEO_FLAGS } from "../inflators/video";
 
 const videoMenuQuery = defineQuery([VideoMenu]);
 const hoverRightVideoQuery = defineQuery([HoveredRemoteRight, MediaVideo]);
@@ -71,12 +72,14 @@ export function videoMenuSystem(world: HubsWorld, userinput: any) {
   }
 
   hoverRightVideoEnterQuery(world).forEach(function (eid) {
-    const menu = rightVideoMenu;
-    VideoMenu.videoRef[menu] = eid;
-    const menuObj = world.eid2obj.get(menu)!;
-    const videoObj = world.eid2obj.get(eid)!;
-    videoObj.add(menuObj);
-    setCursorRaycastable(world, menu, true);
+    if (MediaVideo.flags[eid] & VIDEO_FLAGS.CONTROLS) {
+      const menu = rightVideoMenu;
+      VideoMenu.videoRef[menu] = eid;
+      const menuObj = world.eid2obj.get(menu)!;
+      const videoObj = world.eid2obj.get(eid)!;
+      videoObj.add(menuObj);
+      setCursorRaycastable(world, menu, true);
+    }
   });
 
   videoMenuQuery(world).forEach(function (eid) {

src/bit-systems/video-system.ts

@@ -33,7 +33,7 @@ export function videoSystem(world: HubsWorld, audioSystem: AudioSystem) {
   mediaVideoEnterQuery(world).forEach(function (videoEid) {
     const videoObj = world.eid2obj.get(videoEid) as Mesh;
     const video = MediaVideoData.get(videoEid)!;
-    if (MediaVideo.autoPlay[videoEid]) {
+    if (video.autoplay) {
       video.play().catch(() => {
         // Need to deal with the fact play() may fail if user has not interacted with browser yet.
         console.error("Error auto-playing video.");

src/inflators/image-loader.ts

@@ -1,12 +1,22 @@
 import { HubsWorld } from "../app";
-import { ProjectionMode } from "../utils/projection-mode";
+import { ProjectionModeName } from "../utils/projection-mode";
 import { inflateMediaLoader } from "./media-loader";
+import { MediaImageLoaderData } from "../bit-components";
+import { AlphaModeName } from "../utils/create-image-mesh";
 
 export interface ImageLoaderParams {
   src: string;
-  projection: ProjectionMode;
+  projection: ProjectionModeName;
+  alphaMode: AlphaModeName;
+  alphaCutoff: number;
 }
 
+const DEFAULTS: Partial<ImageLoaderParams> = {
+  projection: ProjectionModeName.FLAT,
+  alphaMode: AlphaModeName.OPAQUE,
+  alphaCutoff: 0.5
+};
+
 export function inflateImageLoader(world: HubsWorld, eid: number, params: ImageLoaderParams) {
   inflateMediaLoader(world, eid, {
     src: params.src,
@@ -16,5 +26,6 @@ export function inflateImageLoader(world: HubsWorld, eid: number, params: ImageL
     isObjectMenuTarget: false
   });
 
-  // TODO: Use projection
+  const requiredParams = Object.assign({}, DEFAULTS, params) as Required<ImageLoaderParams>;
+  MediaImageLoaderData.set(eid, requiredParams);
 }

src/inflators/image.ts

@@ -0,0 +1,42 @@
+import { addComponent } from "bitecs";
+import { MediaImage } from "../bit-components";
+import { addObject3DComponent } from "../utils/jsx-entity";
+import { AlphaMode, create360ImageMesh, createImageMesh } from "../utils/create-image-mesh";
+import { ProjectionMode } from "../utils/projection-mode";
+import { Texture } from "three";
+import { HubsWorld } from "../app";
+import { EntityID } from "../utils/networking-types";
+
+export interface ImageParams {
+  cacheKey: string;
+  texture: Texture;
+  ratio: number;
+  projection: ProjectionMode;
+  alphaMode: AlphaMode;
+  alphaCutoff: number;
+}
+
+const DEFAULTS: Partial<ImageParams> = {
+  projection: ProjectionMode.FLAT,
+  alphaMode: AlphaMode.OPAQUE,
+  alphaCutoff: 0.5,
+  ratio: 1
+};
+
+export function inflateImage(world: HubsWorld, eid: EntityID, params: ImageParams) {
+  const { texture, ratio, projection, alphaMode, alphaCutoff, cacheKey } = params;
+  const mesh =
+    projection === ProjectionMode.SPHERE_EQUIRECTANGULAR
+      ? create360ImageMesh(texture, alphaMode, alphaCutoff)
+      : createImageMesh(texture, ratio, alphaMode, alphaCutoff);
+  addObject3DComponent(world, eid, mesh);
+  addComponent(world, MediaImage, eid);
+
+  const requiredParams = Object.assign({}, DEFAULTS, params) as Required<ImageParams>;
+  MediaImage.projection[eid] = requiredParams.projection;
+  MediaImage.alphaMode[eid] = requiredParams.alphaMode;
+  MediaImage.alphaCutoff[eid] = requiredParams.alphaCutoff;
+  MediaImage.cacheKey[eid] = APP.getSid(cacheKey);
+
+  return eid;
+}

src/inflators/video-loader.ts

@@ -1,15 +1,23 @@
 import { HubsWorld } from "../app";
-import { ProjectionMode } from "../utils/projection-mode";
+import { MediaVideoLoaderData } from "../bit-components";
+import { ProjectionModeName } from "../utils/projection-mode";
 import { inflateMediaLoader } from "./media-loader";
 
 export interface VideoLoaderParams {
   src: string;
-  projection: ProjectionMode;
+  projection: ProjectionModeName;
   autoPlay: boolean;
   controls: boolean;
   loop: boolean;
 }
 
+const DEFAULTS: Partial<VideoLoaderParams> = {
+  projection: ProjectionModeName.FLAT,
+  controls: true,
+  autoPlay: true,
+  loop: true
+};
+
 export function inflateVideoLoader(world: HubsWorld, eid: number, params: VideoLoaderParams) {
   inflateMediaLoader(world, eid, {
     src: params.src,
@@ -19,5 +27,6 @@ export function inflateVideoLoader(world: HubsWorld, eid: number, params: VideoL
     isObjectMenuTarget: false
   });
 
-  // TODO: Use the rest of VideoLoaderParams
+  const requiredParams = Object.assign({}, DEFAULTS, params) as Required<VideoLoaderParams>;
+  MediaVideoLoaderData.set(eid, requiredParams);
 }

src/inflators/video.ts

@@ -0,0 +1,46 @@
+import { create360ImageMesh, createImageMesh } from "../utils/create-image-mesh";
+import { addComponent } from "bitecs";
+import { addObject3DComponent } from "../utils/jsx-entity";
+import { ProjectionMode } from "../utils/projection-mode";
+import { MediaVideo, MediaVideoData } from "../bit-components";
+import { HubsWorld } from "../app";
+import { EntityID } from "../utils/networking-types";
+import { Texture } from "three";
+
+export const VIDEO_FLAGS = {
+  CONTROLS: 1 << 0
+};
+
+export interface VideoParams {
+  texture: Texture;
+  ratio: number;
+  projection: ProjectionMode;
+  video: HTMLVideoElement;
+  controls: boolean;
+}
+
+const DEFAULTS: Partial<VideoParams> = {
+  projection: ProjectionMode.FLAT,
+  controls: true,
+  ratio: 1
+};
+
+export function inflateVideo(world: HubsWorld, eid: EntityID, params: VideoParams) {
+  const { texture, ratio, projection, video } = params;
+  const mesh =
+    projection === ProjectionMode.SPHERE_EQUIRECTANGULAR
+      ? create360ImageMesh(texture)
+      : createImageMesh(texture, ratio);
+  addObject3DComponent(world, eid, mesh);
+  addComponent(world, MediaVideo, eid);
+
+  const requiredParams = Object.assign({}, DEFAULTS, params) as Required<VideoParams>;
+  MediaVideo.flags[eid] = 0;
+  if (!!requiredParams.controls) {
+    MediaVideo.flags[eid] |= VIDEO_FLAGS.CONTROLS;
+  }
+  MediaVideo.projection[eid] = requiredParams.projection;
+  MediaVideo.ratio[eid] = requiredParams.ratio;
+  MediaVideoData.set(eid, video);
+  return eid;
+}

src/inflators/waypoint.tsx

@@ -70,7 +70,7 @@ export function inflateWaypoint(world: HubsWorld, eid: number, props: WaypointPa
             texture,
             ratio: 1,
             projection: ProjectionMode.FLAT,
-            alphaMode: AlphaMode.Blend,
+            alphaMode: AlphaMode.BLEND,
             cacheKey
           }}
         ></entity>

src/prefabs/error-object.js

@@ -13,7 +13,7 @@ export function ErrorObject() {
         texture: loadTextureFromCache("error", 1).texture,
         ratio: 1400 / 1200,
         projection: ProjectionMode.FLAT,
-        alphaMode: AlphaMode.Blend,
+        alphaMode: AlphaMode.BLEND,
         cacheKey: TextureCache.key("error", 1)
       }}
     />

src/prefabs/video-menu.tsx

@@ -72,7 +72,7 @@ export function VideoMenuPrefab() {
           texture: playTexture,
           ratio: 1,
           projection: ProjectionMode.FLAT,
-          alphaMode: AlphaMode.Blend,
+          alphaMode: AlphaMode.BLEND,
           cacheKey: TextureCache.key(playImageUrl, 1)
         }}
         visible={false}
@@ -85,7 +85,7 @@ export function VideoMenuPrefab() {
           texture: pauseTexture,
           ratio: 1,
           projection: ProjectionMode.FLAT,
-          alphaMode: AlphaMode.Blend,
+          alphaMode: AlphaMode.BLEND,
           cacheKey: TextureCache.key(pauseImageUrl, 1)
         }}
         visible={false}