diff --git a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/WindowWidget.java b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/WindowWidget.java
index a1e49b537..f358a29e4 100644
--- a/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/WindowWidget.java
+++ b/app/src/common/shared/org/mozilla/vrbrowser/ui/widgets/WindowWidget.java
@@ -639,6 +639,16 @@ public void setActiveWindow(boolean active) {
         updateBorder();
 
         mViewModel.setIsActiveWindow(active);
+
+        // Remove tha back handler in case there is a library view visible, otherwise it gets dismissed
+        // when back is clicked even if other window is focused.
+        if (mView != null) {
+            if (active) {
+                mWidgetManager.pushBackHandler(mBackHandler);
+            } else {
+                mWidgetManager.popBackHandler(mBackHandler);
+            }
+        }
     }
 
     @Nullable