JRE-705 Z-order of child windows is broken on Mac OS

(cherry picked from commit 88cea60)

2 files changed, 85 insertions, 48 deletions

diff --git a/src/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java b/src/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java
index d21f4e222b..3f13ac600e 100644
--- a/src/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java
+++ b/src/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java
@@ -570,72 +570,88 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
     public void setVisible(boolean visible) {
         final long nsWindowPtr = getNSWindowPtr();
 
-        if (owner != null) {
-            if (!visible) {
-                CWrapper.NSWindow.removeChildWindow(owner.getNSWindowPtr(), nsWindowPtr);
-            }
-        }
-
+        // Configure stuff
         updateIconImages();
         updateFocusabilityForAutoRequestFocus(false);
 
-        if (!visible) {
-            // Cancel out the current native state of the window
-            switch (peer.getState()) {
-                case Frame.ICONIFIED:
-                    CWrapper.NSWindow.deminiaturize(nsWindowPtr);
-                    break;
-                case Frame.MAXIMIZED_BOTH:
-                    CWrapper.NSWindow.zoom(nsWindowPtr);
-                    break;
-            }
-        }
+        boolean wasMaximized = isMaximized();
 
-        LWWindowPeer blocker = peer.getBlocker();
-        if (blocker == null || !visible) {
-            // If it ain't blocked, or is being hidden, go regular way
-            if (visible) {
-                CWrapper.NSWindow.makeFirstResponder(nsWindowPtr, contentView.getAWTView());
-                boolean isKeyWindow = CWrapper.NSWindow.isKeyWindow(nsWindowPtr);
-                if (!isKeyWindow) {
-                    CWrapper.NSWindow.makeKeyAndOrderFront(nsWindowPtr);
+        this.visible = visible;
+
+        // Manage the extended state when showing
+        if (visible) {
+            // Apply the extended state as expected in shared code
+            if (target instanceof Frame) {
+                if (!wasMaximized && isMaximized()) {
+                    // setVisible could have changed the native maximized state
+                    deliverZoom(true);
                 } else {
-                    CWrapper.NSWindow.orderFront(nsWindowPtr);
+                    int frameState = ((Frame)target).getExtendedState();
+                    if ((frameState & Frame.ICONIFIED) != 0) {
+                        // Treat all state bit masks with ICONIFIED bit as ICONIFIED state.
+                        frameState = Frame.ICONIFIED;
+                    }
+                    switch (frameState) {
+                        case Frame.ICONIFIED:
+                            CWrapper.NSWindow.miniaturize(nsWindowPtr);
+                            break;
+                        case Frame.MAXIMIZED_BOTH:
+                            maximize();
+                            break;
+                        default: // NORMAL
+                            unmaximize(); // in case it was maximized, otherwise this is a no-op
+                            break;
+                    }
                 }
-            } else {
-                CWrapper.NSWindow.orderOut(nsWindowPtr);
             }
-        } else {
-            // otherwise, put it in a proper z-order
-            CWrapper.NSWindow.orderWindow(nsWindowPtr, CWrapper.NSWindow.NSWindowBelow,
-                    ((CPlatformWindow)blocker.getPlatformWindow()).getNSWindowPtr());
         }
 
+        nativeSynthesizeMouseEnteredExitedEvents();
+
+        // Configure stuff #2
+        updateFocusabilityForAutoRequestFocus(true);
+
+        // Manage parent-child relationship when showing
         if (visible) {
-            // Re-apply the extended state as expected in shared code
-            if (target instanceof Frame) {
-                switch (((Frame)target).getExtendedState()) {
-                    case Frame.ICONIFIED:
-                        CWrapper.NSWindow.miniaturize(nsWindowPtr);
-                        break;
-                    case Frame.MAXIMIZED_BOTH:
-                        CWrapper.NSWindow.zoom(nsWindowPtr);
-                        break;
-                }
+            // Order myself above my parent
+            if (owner != null && owner.isVisible()) {
+                CWrapper.NSWindow.orderWindow(nsWindowPtr, CWrapper.NSWindow.NSWindowAbove, owner.getNSWindowPtr());
+                applyWindowLevel(target);
             }
         }
 
-        updateFocusabilityForAutoRequestFocus(true);
 
-        if (owner != null) {
+        // Actually show or hide the window
+        LWWindowPeer blocker = (peer == null)? null : peer.getBlocker();
+        if (blocker == null || !visible) {
+
+            // If it ain't blocked, or is being hidden, go regular way
             if (visible) {
-                CWrapper.NSWindow.addChildWindow(owner.getNSWindowPtr(), nsWindowPtr, CWrapper.NSWindow.NSWindowAbove);
-                if (target.isAlwaysOnTop()) {
-                    CWrapper.NSWindow.setLevel(nsWindowPtr, CWrapper.NSWindow.NSFloatingWindowLevel);
+
+                CWrapper.NSWindow.makeFirstResponder(nsWindowPtr, contentView.getAWTView());
+
+                boolean isPopup = (target.getType() == Window.Type.POPUP);
+                if (isPopup) {
+                    // Popups in applets don't activate applet's process
+                    CWrapper.NSWindow.orderFrontRegardless(nsWindowPtr);
+                } else {
+                    CWrapper.NSWindow.orderFront(nsWindowPtr);
+                }
+
+                boolean isKeyWindow = CWrapper.NSWindow.isKeyWindow(nsWindowPtr);
+                if (!isKeyWindow) {
+                  CWrapper.NSWindow.makeKeyWindow(nsWindowPtr);
                 }
+            } else {
+                // immediately hide the window
+                CWrapper.NSWindow.orderOut(nsWindowPtr);
+                // process the close
+                CWrapper.NSWindow.close(nsWindowPtr);
             }
         }
 
+        // Order my own children above myself
+        // Deal with the blocker of the window being shown
         if (blocker != null && visible) {
             // Make sure the blocker is above its siblings
             ((CPlatformWindow)blocker.getPlatformWindow()).orderAboveSiblings();
@@ -1097,7 +1113,10 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
         if (target.isAlwaysOnTop() && target.getType() != Window.Type.POPUP) {
             CWrapper.NSWindow.setLevel(getNSWindowPtr(), CWrapper.NSWindow.NSFloatingWindowLevel);
         } else if (target.getType() == Window.Type.POPUP) {
-            CWrapper.NSWindow.setLevel(getNSWindowPtr(), CWrapper.NSWindow.NSPopUpMenuWindowLevel);
+            CWrapper.NSWindow.setLevel(getNSWindowPtr(),
+                    WindowPeer.isLightweightDialog(target) ?
+                            CWrapper.NSWindow.NSNormalWindowLevel :
+                            CWrapper.NSWindow.NSPopUpMenuWindowLevel);
         }
     }
 
diff --git a/src/share/classes/java/awt/peer/WindowPeer.java b/src/share/classes/java/awt/peer/WindowPeer.java
index d9b2ffd35a..349ef4c18a 100644
--- a/src/share/classes/java/awt/peer/WindowPeer.java
+++ b/src/share/classes/java/awt/peer/WindowPeer.java
@@ -25,6 +25,7 @@
 
 package java.awt.peer;
 
+import javax.swing.*;
 import java.awt.*;
 
 /**
@@ -124,4 +125,21 @@ public interface WindowPeer extends ContainerPeer {
      * @return the system insets or null
      */
     default Insets getSysInsets() { return null; }
+
+    static boolean isLightweightDialog(Window window) {
+        if (window instanceof RootPaneContainer) {
+            RootPaneContainer rootPaneContainer = (RootPaneContainer) window;
+            JRootPane rootPane = rootPaneContainer.getRootPane();
+            if (rootPane != null) {
+                Object property = rootPane.getClientProperty("SIMPLE_WINDOW");
+                if (property instanceof Boolean) {
+                    Boolean isLightweightDialog = (Boolean)property;
+                    if (isLightweightDialog) {
+                        return true;
+                    }
+                }
+            }
+        }
+        return false;
+    }
 }