Skip to content

Commit

Permalink
WebXR: experimental DOM Overlay support for immersive-ar mode
Browse files Browse the repository at this point in the history 
This opt-in mode keeps DOM content visible as a transparent overlay while
in an immersive-ar WebXR session. It is activated by requesting an optional
or required feature on session start:

  navigator.xr.requestSession(
    'immersive-ar',
    {optionalFeatures: ['dom-overlay-for-handheld-ar']});

This functionality is only available if the corresponding feature flag
chrome://flags#webxr-ar-dom-overlay is enabled.

On session start, this fullscreens the <body> element. The application can
use the Fullscreen API to change the visible element. Exiting the session ends
fullscreen mode, and calling document.exitFullscreen() exits the immersive-ar
session if there are no remaining fullscreened elements.

(As of this CL, changing the fullscreen element doesn't fully update layer
visibility, so non-fullscreen content can remain visible unexpectedly.
That's being addressed in a followup.)

Change-Id: I77b767b111436b45e2b584e46a390a68473ab118
Bug: 991747
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1741008
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Xianzhu Wang <wangxianzhu@chromium.org>
Reviewed-by: Philip Jägenstedt <foolip@chromium.org>
Reviewed-by: Alex Moshchuk <alexmos@chromium.org>
Reviewed-by: Rune Lillesveen <futhark@chromium.org>
Reviewed-by: Michael Thiessen <mthiesse@chromium.org>
Reviewed-by: Matthew Jones <mdjones@chromium.org>
Reviewed-by: Alexander Cooper <alcooper@chromium.org>
Commit-Queue: Klaus Weidner <klausw@chromium.org>
Cr-Commit-Position: refs/heads/master@{#703074}
  • Loading branch information
@klausw
klausw authored and Commit Bot committed Oct 4, 2019
1 parent ee784aa commit 38db9a1
Show file tree
Hide file tree
Showing 46 changed files with 524 additions and 45 deletions.
4 changes: 4 additions & 0 deletions chrome/android/java/src/org/chromium/chrome/browser/ChromeActivity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2016,6 +2016,10 @@ public final void onBackPressed() {

TextBubble.dismissBubbles();
if (VrModuleProvider.getDelegate().onBackPressed()) return;

ArDelegate arDelegate = ArDelegateProvider.getDelegate();
if (arDelegate != null && arDelegate.onBackPressed()) return;

if (mCompositorViewHolder != null) {
LayoutManager layoutManager = mCompositorViewHolder.getLayoutManager();
if (layoutManager != null && layoutManager.onBackPressed()) return;
Expand Down
12 changes: 12 additions & 0 deletions chrome/android/java/src/org/chromium/chrome/browser/compositor/CompositorView.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,16 @@ public void setOverlayVideoMode(boolean enabled) {
// We do know that if we do get one, then it will be for the surface that we just requested.
}

/**
* Enables/disables immersive AR overlay mode, a variant of overlay video mode.
* @param enabled Whether to enter or leave overlay immersive ar mode.
*/
public void setOverlayImmersiveArMode(boolean enabled) {
setOverlayVideoMode(enabled);
CompositorViewJni.get().setOverlayImmersiveArMode(
mNativeCompositorView, CompositorView.this, enabled);
}

private int getSurfacePixelFormat() {
if (mOverlayVideoEnabled || mAlwaysTranslucent) {
return PixelFormat.TRANSLUCENT;
Expand Down Expand Up @@ -620,6 +630,8 @@ void onPhysicalBackingSizeChanged(long nativeCompositorView, CompositorView call
void setNeedsComposite(long nativeCompositorView, CompositorView caller);
void setLayoutBounds(long nativeCompositorView, CompositorView caller);
void setOverlayVideoMode(long nativeCompositorView, CompositorView caller, boolean enabled);
void setOverlayImmersiveArMode(
long nativeCompositorView, CompositorView caller, boolean enabled);
void setSceneLayer(long nativeCompositorView, CompositorView caller, SceneLayer sceneLayer);
void setCompositorWindow(
long nativeCompositorView, CompositorView caller, WindowAndroid window);
Expand Down
42 changes: 37 additions & 5 deletions chrome/android/java/src/org/chromium/chrome/browser/vr/ArCoreJavaUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@

/**
* Provides ARCore classes access to java-related app functionality.
*
* <p>This class provides static methods called by ArDelegateImpl via ArDelegateProvider,
* and provides JNI interfaces to/from the C++ AR code.</p>
*/
@JNINamespace("vr")
public class ArCoreJavaUtils {
Expand All @@ -28,8 +31,19 @@ public class ArCoreJavaUtils {

private long mNativeArCoreJavaUtils;

// The native ArCoreDevice runtime creates a ArCoreJavaUtils instance in its constructor,
// and keeps a strong reference to it for the lifetime of the device. It creates and
// owns an ArImmersiveOverlay for the duration of an immersive-ar session, which in
// turn contains a reference to ArCoreJavaUtils for making JNI calls back to the device.
private ArImmersiveOverlay mArImmersiveOverlay;

// ArDelegateImpl needs to know if there's an active immersive session so that it can handle
// back button presses from ChromeActivity's onBackPressed(). It's only set while a session is
// in progress, and reset to null on session end. The ArImmersiveOverlay member has a strong
// reference to the ChromeActivity, and that shouldn't be retained beyond the duration of a
// session.
private static ArCoreJavaUtils sActiveSessionInstance;

@CalledByNative
private static ArCoreJavaUtils create(long nativeArCoreJavaUtils) {
ThreadUtils.assertOnUiThread();
Expand Down Expand Up @@ -60,18 +74,33 @@ private ArCoreJavaUtils(long nativeArCoreJavaUtils) {
}

@CalledByNative
private void startSession(final Tab tab) {
private void startSession(final Tab tab, boolean useOverlay) {
if (DEBUG_LOGS) Log.i(TAG, "startSession");
mArImmersiveOverlay = new ArImmersiveOverlay();
mArImmersiveOverlay.show(tab.getActivity(), this);
sActiveSessionInstance = this;
mArImmersiveOverlay.show(tab.getActivity(), this, useOverlay);
}

@CalledByNative
private void endSession() {
if (DEBUG_LOGS) Log.i(TAG, "endSession");
if (mArImmersiveOverlay != null) {
mArImmersiveOverlay.cleanupAndExit();
if (mArImmersiveOverlay == null) return;

mArImmersiveOverlay.cleanupAndExit();
mArImmersiveOverlay = null;
sActiveSessionInstance = null;
}

// Called from ArDelegateImpl
public static boolean onBackPressed() {
if (DEBUG_LOGS) Log.i(TAG, "onBackPressed");
// If there's an active immersive session, consume the "back" press and shut down the
// session.
if (sActiveSessionInstance != null) {
sActiveSessionInstance.endSession();
return true;
}
return false;
}

public void onDrawingSurfaceReady(Surface surface, int rotation, int width, int height) {
Expand All @@ -91,13 +120,16 @@ public void onDrawingSurfaceTouch(boolean isTouching, float x, float y) {
public void onDrawingSurfaceDestroyed() {
if (DEBUG_LOGS) Log.i(TAG, "onDrawingSurfaceDestroyed");
if (mNativeArCoreJavaUtils == 0) return;
mArImmersiveOverlay = null;
ArCoreJavaUtilsJni.get().onDrawingSurfaceDestroyed(
mNativeArCoreJavaUtils, ArCoreJavaUtils.this);
}

@CalledByNative
private void onNativeDestroy() {
// ArCoreDevice's destructor ends sessions before destroying its native ArCoreSessionUtils
// object.
assert sActiveSessionInstance == null : "unexpected active session in onNativeDestroy";

mNativeArCoreJavaUtils = 0;
}

Expand Down
5 changes: 5 additions & 0 deletions chrome/android/java/src/org/chromium/chrome/browser/vr/ArDelegate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,9 @@ public interface ArDelegate {
* to notify AR that the activity was resumed.
**/
public void registerOnResumeActivity(Activity activity);

/**
* Used to let AR immersive mode intercept the Back button to exit immersive mode.
*/
public boolean onBackPressed();
}
5 changes: 5 additions & 0 deletions chrome/android/java/src/org/chromium/chrome/browser/vr/ArDelegateImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,9 @@ public void init() {
public void registerOnResumeActivity(Activity activity) {
ArCoreInstallUtils.onResumeActivityWithNative(activity);
}

@Override
public boolean onBackPressed() {
return ArCoreJavaUtils.onBackPressed();
}
}
120 changes: 108 additions & 12 deletions chrome/android/java/src/org/chromium/chrome/browser/vr/ArImmersiveOverlay.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,13 @@
import android.app.Dialog;
import android.content.DialogInterface;
import android.content.pm.ActivityInfo;
import android.graphics.PixelFormat;
import android.graphics.Point;
import android.view.Display;
import android.view.Gravity;
import android.view.MotionEvent;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.View;
import android.view.ViewGroup;

Expand All @@ -19,6 +23,9 @@
import org.chromium.base.Log;
import org.chromium.chrome.R;
import org.chromium.chrome.browser.ChromeActivity;
import org.chromium.chrome.browser.compositor.CompositorView;
import org.chromium.chrome.browser.tab.EmptyTabObserver;
import org.chromium.chrome.browser.tab.Tab;
import org.chromium.content_public.browser.ScreenOrientationDelegate;
import org.chromium.content_public.browser.ScreenOrientationProvider;
import org.chromium.ui.widget.Toast;
Expand All @@ -38,14 +45,19 @@ public class ArImmersiveOverlay
private boolean mCleanupInProgress;
private SurfaceUiWrapper mSurfaceUi;

public void show(@NonNull ChromeActivity activity, @NonNull ArCoreJavaUtils caller) {
public void show(
@NonNull ChromeActivity activity, @NonNull ArCoreJavaUtils caller, boolean useOverlay) {
if (DEBUG_LOGS) Log.i(TAG, "constructor");
mArCoreJavaUtils = caller;
mActivity = activity;

// Choose a concrete implementation to create a drawable Surface and make it fullscreen.
// It forwards SurfaceHolder callbacks and touch events to this ArImmersiveOverlay object.
mSurfaceUi = new SurfaceUiDialog(this);
if (useOverlay) {
mSurfaceUi = new SurfaceUiCompositor();
} else {
mSurfaceUi = new SurfaceUiDialog();
}
}

private interface SurfaceUiWrapper {
Expand All @@ -64,14 +76,14 @@ private class SurfaceUiDialog implements SurfaceUiWrapper, DialogInterface.OnCan
| View.SYSTEM_UI_FLAG_FULLSCREEN | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION
| View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY;

public SurfaceUiDialog(ArImmersiveOverlay parent) {
public SurfaceUiDialog() {
// Create a fullscreen dialog and use its backing Surface for drawing.
mDialog = new Dialog(mActivity, android.R.style.Theme_NoTitleBar_Fullscreen);
mDialog.getWindow().setBackgroundDrawable(null);
mDialog.getWindow().takeSurface(parent);
mDialog.getWindow().takeSurface(ArImmersiveOverlay.this);
View view = mDialog.getWindow().getDecorView();
view.setSystemUiVisibility(VISIBILITY_FLAGS_IMMERSIVE);
view.setOnTouchListener(parent);
view.setOnTouchListener(ArImmersiveOverlay.this);
mDialog.setOnCancelListener(this);
mDialog.getWindow().setLayout(
ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT);
Expand Down Expand Up @@ -103,6 +115,52 @@ public void onCancel(DialogInterface dialog) {
}
}

private class SurfaceUiCompositor extends EmptyTabObserver implements SurfaceUiWrapper {
private SurfaceView mSurfaceView;
private CompositorView mCompositorView;

public SurfaceUiCompositor() {
mSurfaceView = new SurfaceView(mActivity);
// Keep the camera layer at "default" Z order. Chrome's compositor SurfaceView is in
// OverlayVideoMode, putting it in front of that, but behind other non-SurfaceView UI.
mSurfaceView.getHolder().setFormat(PixelFormat.TRANSLUCENT);
mSurfaceView.getHolder().addCallback(ArImmersiveOverlay.this);

View content = mActivity.getWindow().findViewById(android.R.id.content);
ViewGroup group = (ViewGroup) content.getParent();
group.addView(mSurfaceView);

mCompositorView = mActivity.getCompositorViewHolder().getCompositorView();

// Enable alpha channel for the compositor and make the background
// transparent. (A variant of CompositorView::SetOverlayVideoMode.)
if (DEBUG_LOGS) Log.i(TAG, "calling mCompositorView.setOverlayImmersiveArMode(true)");
mCompositorView.setOverlayImmersiveArMode(true);

// Watch for fullscreen exit triggered from JS, this needs to end the session.
mActivity.getActivityTab().addObserver(this);
}

@Override // SurfaceUiWrapper
public void onSurfaceVisible() {}

@Override // SurfaceUiWrapper
public void destroy() {
mActivity.getActivityTab().removeObserver(this);
View content = mActivity.getWindow().findViewById(android.R.id.content);
ViewGroup group = (ViewGroup) content.getParent();
group.removeView(mSurfaceView);
mSurfaceView = null;
mCompositorView.setOverlayImmersiveArMode(false);
}

@Override // TabObserver
public void onExitFullscreenMode(Tab tab) {
if (DEBUG_LOGS) Log.i(TAG, "onExitFullscreenMode");
cleanupAndExit();
}
}

@Override // View.OnTouchListener
public boolean onTouch(View v, MotionEvent ev) {
// Only forward primary actions, ignore more complex events such as secondary pointer
Expand Down Expand Up @@ -143,32 +201,70 @@ public void surfaceRedrawNeeded(SurfaceHolder holder) {

@Override // SurfaceHolder.Callback2
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
// WebXR immersive sessions don't support resize, so use the first reported size.
// We shouldn't get resize events since we're using FLAG_LAYOUT_STABLE and are
// locking screen orientation.
// The surface may not immediately start out at the expected fullscreen size due to
// animations or not-yet-hidden navigation bars. WebXR immersive sessions use a fixed-size
// frame transport that can't be resized, so we need to pick a single size and stick with it
// for the duration of the session. Use the expected fullscreen size for WebXR frame
// transport even if the currently-visible part in the surface view is smaller than this. We
// shouldn't get resize events since we're using FLAG_LAYOUT_STABLE and are locking screen
// orientation.
if (mSurfaceReportedReady) {
int rotation = mActivity.getWindowManager().getDefaultDisplay().getRotation();
if (DEBUG_LOGS)
if (DEBUG_LOGS) {
Log.i(TAG,
"surfaceChanged ignoring change to width=" + width + " height=" + height
+ " rotation=" + rotation);
}
return;
}

// Save current orientation mode, and then lock current orientation.
// Need to ensure orientation is locked at this point to avoid race conditions. Save current
// orientation mode, and then lock current orientation. It's unclear if there's still a risk
// of races, for example if an orientation change was already in progress at this point but
// wasn't fully processed yet. In that case the user may need to exit and re-enter the
// session to get the intended layout.
ScreenOrientationProvider.getInstance().setOrientationDelegate(this);
if (mRestoreOrientation == null) {
mRestoreOrientation = mActivity.getRequestedOrientation();
}
mActivity.setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LOCKED);

// Display.getRealSize "gets the real size of the display without subtracting any window
// decor or applying any compatibility scale factors", and "the size is adjusted based on
// the current rotation of the display". This is what we want since the surface and WebXR
// frame sizes also use the same current rotation which is now locked, so there's no need to
// separately adjust for portrait vs landscape modes.
//
// While it would be preferable to wait until the surface is at the desired fullscreen
// resolution, i.e. via mActivity.getFullscreenManager().getPersistentFullscreenMode(), that
// causes a chicken-and-egg problem for SurfaceUiCompositor mode as used for DOM overlay.
// Chrome's fullscreen mode is triggered by the Blink side setting an element fullscreen
// after the session starts, but the session doesn't start until we report the drawing
// surface being ready (including a configured size), so we use this reported size assuming
// that's what the fullscreen mode will use.
Display display = mActivity.getWindowManager().getDefaultDisplay();
Point size = new Point();
display.getRealSize(size);

if (width < size.x || height < size.y) {
if (DEBUG_LOGS) {
Log.i(TAG,
"surfaceChanged adjusting size from " + width + "x" + height + " to "
+ size.x + "x" + size.y);
}
width = size.x;
height = size.y;
}

int rotation = mActivity.getWindowManager().getDefaultDisplay().getRotation();
if (DEBUG_LOGS)
if (DEBUG_LOGS) {
Log.i(TAG, "surfaceChanged size=" + width + "x" + height + " rotation=" + rotation);
}
mArCoreJavaUtils.onDrawingSurfaceReady(holder.getSurface(), rotation, width, height);
mSurfaceReportedReady = true;

// Show a toast with instructions how to exit fullscreen mode now if necessary.
// Show the toast with instructions how to exit fullscreen mode now if necessary.
// Not needed in DOM overlay mode which uses FullscreenHtmlApiHandler to do so.
mSurfaceUi.onSurfaceVisible();
}

Expand Down
3 changes: 3 additions & 0 deletions chrome/browser/about_flags.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2207,6 +2207,9 @@ const FeatureEntry kFeatureEntries[] = {
{"webxr-ar-module", flag_descriptions::kWebXrArModuleName,
flag_descriptions::kWebXrArModuleDescription, kOsAll,
FEATURE_VALUE_TYPE(features::kWebXrArModule)},
{"webxr-ar-dom-overlay", flag_descriptions::kWebXrArDOMOverlayName,
flag_descriptions::kWebXrArDOMOverlayDescription, kOsAndroid,
FEATURE_VALUE_TYPE(features::kWebXrArDOMOverlay)},
{"webxr-hit-test", flag_descriptions::kWebXrHitTestName,
flag_descriptions::kWebXrHitTestDescription, kOsAll,
FEATURE_VALUE_TYPE(features::kWebXrHitTest)},
Expand Down
Loading

0 comments on commit 38db9a1

Please sign in to comment.