Augmented faces sample (#93)

Augmented Face
- AugmentedFaceNode
- FaceArFragment
- AugmentedFace Sample
- Texture names should only be set once on a GL thread

Co-authored-by: Nikita Zaytsev <[email protected]>
Co-authored-by: ThomasGorisse <[email protected]>

core/assets-sources/materials/ar_environment_material_depth.mat

@@ -2,6 +2,7 @@ material {
     name : depth,
     shadingModel : unlit,
     blending : opaque,
+    culling : none,
     vertexDomain : device,
     parameters : [
         {

core/assets-sources/materials/ar_environment_material_flat.mat

@@ -2,6 +2,7 @@ material {
     name : flat,
     shadingModel : unlit,
     blending : opaque,
+    culling : none,
     parameters : [
         {
             type : samplerExternal,

core/src/main/java/com/google/ar/sceneform/ArSceneView.java

@@ -7,6 +7,8 @@
 import android.view.Display;
 import android.view.WindowManager;
 
+import androidx.annotation.IntRange;
+import androidx.annotation.IntRange;
 import androidx.annotation.Nullable;
 import androidx.annotation.UiThread;
 
@@ -55,6 +57,7 @@ public class ArSceneView extends SceneView {
     // threads however.
     private final SequentialTask pauseResumeTask = new SequentialTask();
     private int cameraTextureId;
+    private boolean hasSetTextureNames = false;
     @Nullable
     private Session session;
 
@@ -439,6 +442,14 @@ protected boolean onBeginFrame(long frameTimeNanos) {
         // Before doing anything update the Frame from ARCore.
         boolean arFrameUpdated = true;
         try {
+            // Texture names should only be set once on a GL thread unless they change.
+            // This is done during onDrawFrame rather than onSurfaceCreated since the session is
+            // not guaranteed to have been initialized during the execution of onSurfaceCreated.
+            if (!hasSetTextureNames) {
+                session.setCameraTextureName(cameraTextureId);
+                hasSetTextureNames = true;
+            }
+
             Frame frame = session.update();
             // No frame, no drawing.
             if (frame == null) {
@@ -544,6 +555,10 @@ private void initializeAr() {
         cameraStream = new CameraStream(cameraTextureId, renderer);
     }
 
+    public void setCameraStreamRenderPriority(@IntRange(from = 0L, to = 7L) int priority) {
+        this.cameraStream.setRenderPriority(priority);
+    }
+
     //
     // TODO : When Kotlining it, move all those trackables parts to Trackables.kt as an ArSceneView extension.
     //

core/src/main/java/com/google/ar/sceneform/rendering/LoadRenderableFromSfbTask.java

@@ -426,7 +426,7 @@ private CompletableFuture<SceneformBundleDef> loadTexturesAsync(SceneformBundleD
         ByteArrayInputStream wrappedInputStream =
             new ByteArrayInputStream(data.array(), data.arrayOffset(), data.capacity());
         // position the stream to the image buffer
-        boolean premultiplyAlpha = (usage == Texture.Usage.COLOR);
+        boolean premultiplyAlpha = (usage == Texture.Usage.COLOR_MAP);
         wrappedInputStream.skip(data.position());
         // TODO: The registryId should be populated with a sha1sum
 

core/src/main/java/com/google/ar/sceneform/rendering/Material.java

@@ -157,6 +157,12 @@ public void setTexture(String name, Texture texture) {
         }
     }
 
+    public void setBaseColorTexture(Texture texture) {
+        //Set the baseColorIndex to 0 if no existing texture was set
+        setInt("baseColorIndex", 0);
+        setTexture("baseColorMap", texture);
+    }
+
     /**
      * <pre>
      *     Sets a {@link DepthTexture} to a parameter of the type 'sampler2d' on this material.

core/src/main/java/com/google/ar/sceneform/rendering/Texture.java

@@ -28,10 +28,10 @@ public class Texture {
   /** Type of Texture usage. */
   public enum Usage {
     /** Texture contains a color map */
-    COLOR,
+    COLOR_MAP,
     /** Assume color usage by default */
     /** Texture contains a normal map */
-    NORMAL,
+    NORMAL_MAP,
     /** Texture contains arbitrary data */
     DATA
   }
@@ -77,10 +77,10 @@ private static com.google.android.filament.Texture.InternalFormat getInternalFor
     com.google.android.filament.Texture.InternalFormat format;
 
     switch (usage) {
-      case COLOR:
+      case COLOR_MAP:
         format = com.google.android.filament.Texture.InternalFormat.SRGB8_A8;
         break;
-      case NORMAL:
+      case NORMAL_MAP:
       case DATA:
       default:
         format = com.google.android.filament.Texture.InternalFormat.RGBA8;
@@ -97,7 +97,7 @@ public static final class Builder {
     @Nullable private Bitmap bitmap = null;
     @Nullable private TextureInternalData textureInternalData = null;
 
-    private Usage usage = Usage.COLOR;
+    private Usage usage = Usage.COLOR_MAP;
     /** Enables reuse through the registry */
     @Nullable private Object registryId = null;
 

samples/augmented-faces/.gitignore

@@ -0,0 +1 @@
+/build

samples/augmented-faces/build.gradle

@@ -0,0 +1,34 @@
+apply plugin: 'com.android.application'
+
+android {
+    compileSdkVersion 30
+    defaultConfig {
+        applicationId "com.google.ar.sceneform.samples.augmentedfaces"
+
+        // Sceneform requires minSdkVersion >= 24.
+        minSdkVersion 24
+        targetSdkVersion 30
+        versionCode 1
+        versionName "1.0"
+    }
+    // Sceneform libraries use language constructs from Java 8.
+    // Add these compile options if targeting minSdkVersion < 26.
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+    }
+}
+
+dependencies {
+    implementation 'androidx.appcompat:appcompat:1.2.0'
+    implementation 'androidx.fragment:fragment:1.3.2'
+    implementation 'com.google.android.material:material:1.3.0'
+
+    api project(":sceneform")
+}

samples/augmented-faces/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

samples/augmented-faces/src/main/AndroidManifest.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.ar.sceneform.samples.augmentedfaces">
+
+    <!-- Always needed for AR. -->
+    <uses-permission android:name="android.permission.CAMERA" />
+
+    <!-- Needed to load gltf from network. -->
+    <uses-permission android:name="android.permission.INTERNET" />
+
+    <!-- Sceneform requires OpenGLES 3.0 or later. -->
+    <uses-feature
+        android:glEsVersion="0x00030000"
+        android:required="true" />
+
+    <!-- Indicates that this app requires Google Play Services for AR ("AR Required") and results in
+         the app only being visible in the Google Play Store on devices that support ARCore.
+         For an "AR Optional" app, remove this tag. -->
+    <uses-feature
+        android:name="android.hardware.camera.ar"
+        android:required="true" />
+
+    <application
+        android:icon="@drawable/ic_launcher"
+        android:label="@string/app_name"
+        android:theme="@style/AppTheme">
+
+        <!-- Indicates that this app requires Google Play Services for AR ("AR Required") and causes
+        the Google Play Store to download and intall Google Play Services for AR along with the app.
+        For an "AR Optional" app, specify "optional" instead of "required". -->
+        <meta-data
+            android:name="com.google.ar.core"
+            android:value="required" />
+
+        <activity
+            android:name=".MainActivity"
+            android:configChanges="orientation|screenSize"
+            android:exported="true"
+            android:label="@string/app_name"
+            android:screenOrientation="locked"
+            android:theme="@style/AppTheme">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+</manifest>

samples/augmented-faces/src/main/java/com/google/ar/sceneform/samples/augmentedfaces/MainActivity.java

@@ -0,0 +1,160 @@
+package com.google.ar.sceneform.samples.augmentedfaces;
+
+import android.net.Uri;
+import android.os.Bundle;
+import android.widget.Toast;
+
+import androidx.annotation.NonNull;
+import androidx.appcompat.app.AppCompatActivity;
+import androidx.fragment.app.Fragment;
+import androidx.fragment.app.FragmentManager;
+
+import com.google.ar.core.AugmentedFace;
+import com.google.ar.core.Frame;
+import com.google.ar.sceneform.ArSceneView;
+import com.google.ar.sceneform.FrameTime;
+import com.google.ar.sceneform.Sceneform;
+import com.google.ar.sceneform.rendering.ModelRenderable;
+import com.google.ar.sceneform.rendering.Renderable;
+import com.google.ar.sceneform.rendering.RenderableInstance;
+import com.google.ar.sceneform.rendering.Texture;
+import com.google.ar.sceneform.ux.ArFragment;
+import com.google.ar.sceneform.ux.AugmentedFaceNode;
+import com.google.ar.sceneform.ux.FaceArFragment;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+
+public class MainActivity extends AppCompatActivity {
+
+    private final Set<CompletableFuture<?>> loaders = new HashSet<>();
+
+    private FaceArFragment arFragment;
+    private ArSceneView arSceneView;
+
+    private Texture faceTexture;
+    private ModelRenderable faceModel;
+
+    private final HashMap<AugmentedFace, AugmentedFaceNode> facesNodes = new HashMap<>();
+
+    @Override
+    protected void onCreate(Bundle savedInstanceState) {
+        super.onCreate(savedInstanceState);
+
+        setContentView(R.layout.activity_main);
+
+        getSupportFragmentManager().addFragmentOnAttachListener(this::onAttachFragment);
+
+        if (savedInstanceState == null) {
+            if (Sceneform.isSupported(this)) {
+                getSupportFragmentManager().beginTransaction()
+                        .add(R.id.arFragment, FaceArFragment.class, null)
+                        .commit();
+            }
+        }
+
+        loadModels();
+        loadTextures();
+    }
+
+    public void onAttachFragment(@NonNull FragmentManager fragmentManager, @NonNull Fragment fragment) {
+        if (fragment.getId() == R.id.arFragment) {
+            arFragment = (FaceArFragment) fragment;
+            arFragment.setOnViewCreatedListener(this::onViewCreated);
+        }
+    }
+
+    public void onViewCreated(ArFragment arFragment, ArSceneView arSceneView) {
+        this.arSceneView = arSceneView;
+
+        // This is important to make sure that the camera stream renders first so that
+        // the face mesh occlusion works correctly.
+        arSceneView.setCameraStreamRenderPriority(Renderable.RENDER_PRIORITY_FIRST);
+
+        // Check for face detections
+        arSceneView.getScene().addOnUpdateListener(this::onUpdate);
+    }
+
+    @Override
+    protected void onDestroy() {
+        super.onDestroy();
+
+        for (CompletableFuture<?> loader : loaders) {
+            if (!loader.isDone()) {
+                loader.cancel(true);
+            }
+        }
+    }
+
+    private void loadModels() {
+        loaders.add(ModelRenderable.builder()
+                .setSource(this, Uri.parse("models/fox.glb"))
+                .setIsFilamentGltf(true)
+                .build()
+                .thenAccept(model -> faceModel = model)
+                .exceptionally(throwable -> {
+                    Toast.makeText(this, "Unable to load renderable", Toast.LENGTH_LONG).show();
+                    return null;
+                }));
+    }
+
+    private void loadTextures() {
+        loaders.add(Texture.builder()
+                .setSource(this, Uri.parse("textures/freckles.png"))
+                .setUsage(Texture.Usage.COLOR_MAP)
+                .build()
+                .thenAccept(texture -> faceTexture = texture)
+                .exceptionally(throwable -> {
+                    Toast.makeText(this, "Unable to load texture", Toast.LENGTH_LONG).show();
+                    return null;
+                }));
+    }
+
+    private void onUpdate(FrameTime frameTime) {
+        if (faceModel == null || faceTexture == null) {
+            return;
+        }
+
+        Frame frame = arFragment.getArSceneView().getArFrame();
+
+        // Get a list of AugmentedFace which are updated on this frame.
+        Collection<AugmentedFace> augmentedFaces = frame.getUpdatedTrackables(AugmentedFace.class);
+
+        // TODO: Check the difference with getAllTrackables.
+        //  See: https://stackoverflow.com/questions/49241526/what-is-the-difference-between-session-getalltrackables-and-frame-getupdatedtrac
+        //  Collection<AugmentedFace> augmentedFaces = arSceneView.getSession().getAllTrackables(AugmentedFace.class);
+
+        // Make new AugmentedFaceNodes for any new faces.
+        for (AugmentedFace augmentedFace : new ArrayList<>(augmentedFaces)) {
+            AugmentedFaceNode existingFaceNode = facesNodes.get(augmentedFace);
+
+            switch (augmentedFace.getTrackingState()) {
+                case TRACKING:
+                    if (existingFaceNode == null) {
+                        AugmentedFaceNode faceNode = new AugmentedFaceNode(augmentedFace);
+
+                        RenderableInstance modelInstance = faceNode.setFaceRegionsRenderable(faceModel);
+                        modelInstance.setShadowCaster(false);
+                        modelInstance.setShadowReceiver(true);
+
+                        faceNode.setFaceMeshTexture(faceTexture);
+
+                        arSceneView.getScene().addChild(faceNode);
+
+                        facesNodes.put(augmentedFace, faceNode);
+                    }
+                    break;
+                case STOPPED:
+                    if (existingFaceNode != null) {
+                        arSceneView.getScene().removeChild(existingFaceNode);
+                    }
+                    facesNodes.remove(augmentedFace);
+                    break;
+            }
+        }
+    }
+}