diff --git a/src/core/Billboard.tsx b/src/core/Billboard.tsx
index 8b862fe13..b5b10e714 100644
--- a/src/core/Billboard.tsx
+++ b/src/core/Billboard.tsx
@@ -1,7 +1,6 @@
 import * as React from 'react'
-import { Group } from 'three'
+import { Group, Quaternion } from 'three'
 import { useFrame } from '@react-three/fiber'
-import mergeRefs from 'react-merge-refs'
 import { ForwardRefComponent } from '../helpers/ts-utils'
 
 export type BillboardProps = {
@@ -23,8 +22,11 @@ export type BillboardProps = {
 export const Billboard: ForwardRefComponent<BillboardProps, Group> = /* @__PURE__ */ React.forwardRef<
   Group,
   BillboardProps
->(function Billboard({ follow = true, lockX = false, lockY = false, lockZ = false, ...props }, ref) {
-  const localRef = React.useRef<Group>()
+>(function Billboard({ children, follow = true, lockX = false, lockY = false, lockZ = false, ...props }, fref) {
+  const inner = React.useRef<Group>(null!)
+  const localRef = React.useRef<Group>(null!)
+  const q = new Quaternion()
+
   useFrame(({ camera }) => {
     if (!follow || !localRef.current) return
 
@@ -32,12 +34,21 @@ export const Billboard: ForwardRefComponent<BillboardProps, Group> = /* @__PURE_
     const prevRotation = localRef.current.rotation.clone()
 
     // always face the camera
-    camera.getWorldQuaternion(localRef.current.quaternion)
+    localRef.current.updateMatrix()
+    localRef.current.updateWorldMatrix(false, false)
+    localRef.current.getWorldQuaternion(q)
+    camera.getWorldQuaternion(inner.current.quaternion).premultiply(q.invert())
 
     // readjust any axis that is locked
     if (lockX) localRef.current.rotation.x = prevRotation.x
     if (lockY) localRef.current.rotation.y = prevRotation.y
     if (lockZ) localRef.current.rotation.z = prevRotation.z
   })
-  return <group ref={mergeRefs([localRef, ref])} {...props} />
+
+  React.useImperativeHandle(fref, () => localRef.current, [])
+  return (
+    <group ref={localRef} matrixAutoUpdate={false} matrixWorldAutoUpdate={false} {...props}>
+      <group ref={inner}>{children}</group>
+    </group>
+  )
 })