diff --git a/src/UI/DisplayManager.cs b/src/UI/DisplayManager.cs
index ee5455d..e27385f 100644
--- a/src/UI/DisplayManager.cs
+++ b/src/UI/DisplayManager.cs
@@ -19,6 +19,7 @@ namespace UnityExplorer.UI
         public static int Height => ActiveDisplay.renderingHeight;
 
         public static Vector3 MousePosition => Display.RelativeMouseAt(InputManager.MousePosition);
+        public static bool MouseInTargetDisplay => MousePosition.z == ActiveDisplayIndex;
 
         private static Camera canvasCamera;
 
diff --git a/src/UI/Panels/PanelDragger.cs b/src/UI/Panels/PanelDragger.cs
index 0ce5700..f3e226e 100644
--- a/src/UI/Panels/PanelDragger.cs
+++ b/src/UI/Panels/PanelDragger.cs
@@ -67,6 +67,9 @@ namespace UnityExplorer.UI.Panels
 
         public static void UpdateInstances()
         {
+            if (!DisplayManager.MouseInTargetDisplay)
+                return;
+
             if (!resizeCursorObj)
                 CreateCursorUI();
 
diff --git a/src/UI/Panels/UIPanel.cs b/src/UI/Panels/UIPanel.cs
index 1139203..5e54afc 100644
--- a/src/UI/Panels/UIPanel.cs
+++ b/src/UI/Panels/UIPanel.cs
@@ -33,7 +33,8 @@ namespace UnityExplorer.UI.Panels
                 return;
 
             // if the user is clicking
-            if (InputManager.GetMouseButtonDown(0) || InputManager.GetMouseButtonDown(1))
+            if (DisplayManager.MouseInTargetDisplay 
+                && (InputManager.GetMouseButtonDown(0) || InputManager.GetMouseButtonDown(1)))
             {
                 int count = UIManager.PanelHolder.transform.childCount;
                 var mousePos = DisplayManager.MousePosition;