setModal issue with 2 Jdialogs within a Jframe

I am facing issues when I set my first JDialog modal and the second one non-modal.

This is the functionality I am trying to implement:

On click of "Test the dialog!" button, a JDialog with name Custom Dialog Main will open.
If click "yes" option in Custom Dialog Main, another JDialog named Custom Dialog Search will open.
If click "yes" option in Custom Dialog Search, then Custom Dialog Main should come front.
And I should be able to select any JDialog. For example if I select Custom Dialog Search, the other dialog should go back and vice versa.

Problem I am facing is when I click "yes" in Custom Dialog Main, then Custom Dialog Search is displayed behind the main dialog.

This is happening because I am setting Custom Dialog Search non-modal. If I this dialog modal it is displayed correctly but after I click "yes" Custom Dialog Main doesn't come front.

I even tried to set CustomDialogSearch's parent to be CustomDialog the behaviour still not correct.

Below is the example code I am testing.

import java.awt.event.ActionListener;  
import javax.swing.JFrame;  
import javax.swing.JButton;  
import java.awt.event.WindowAdapter;  
import java.awt.event.WindowEvent;  
import java.awt.event.ActionEvent;  
import java.awt.Dimension;   

public class TestTheDialog implements ActionListener {  
    JFrame mainFrame = null;  
    JButton myButton = null;  

    public TestTheDialog() {  
        mainFrame = new JFrame("TestTheDialog Tester");  
        mainFrame.addWindowListener(new WindowAdapter() {  
                public void windowClosing(WindowEvent e) {System.exit(0);}  
            });  
        myButton = new JButton("Test the dialog!");  
        myButton.addActionListener(this);  
        mainFrame.setLocationRelativeTo(null);  
        mainFrame.getContentPane().add(myButton);  
        mainFrame.pack();  
        mainFrame.setVisible(true);  
    }  

    public void actionPerformed(ActionEvent e) {  
        if(myButton == e.getSource()) {  
            System.err.println("Opening dialog.");  
            CustomDialog myDialog = new CustomDialog(mainFrame, true, "Custom Dialog Main?");  
            System.err.println("After opening dialog.");  
            if(myDialog.getAnswer()) {  
                System.err.println("The answer stored in CustomDialog is 'true' (i.e. user clicked yes button.)");  
            }  
            else {  
                System.err.println("The answer stored in CustomDialog is 'false' (i.e. user clicked no button.)");  
            }  
        }  
    }  

    public static void main(String argv[]) {  

        TestTheDialog tester = new TestTheDialog();  
    }  
}  

import javax.swing.JDialog;   
import java.awt.event.ActionListener;  
import javax.swing.JPanel;  
import javax.swing.JFrame;  
import javax.swing.JLabel;  
import javax.swing.JButton;  
import java.awt.event.ActionEvent;  

public class CustomDialog extends JDialog implements ActionListener {  
    private JPanel myPanel = null;  
    private JButton yesButton = null;  
    private JButton noButton = null;  
    private boolean answer = false;  
    private JFrame parentFrame;  
    public boolean getAnswer() { return answer; }  

    public CustomDialog(JFrame frame, boolean modal, String myMessage) {  
        super(frame, modal);  
        parentFrame = frame;  
        myPanel = new JPanel();  
        getContentPane().add(myPanel);  
        myPanel.add(new JLabel(myMessage));  
        yesButton = new JButton("Yes");  
        yesButton.addActionListener(this);  
        myPanel.add(yesButton);   
        noButton = new JButton("No");  
        noButton.addActionListener(this);  
        myPanel.add(noButton);    
        pack();  
        setLocationRelativeTo(frame);  
        setVisible(true);  
    }  

    public void actionPerformed(ActionEvent e) {  
        if(yesButton == e.getSource()) {  
            CustomDialogSearch myDialog = new CustomDialogSearch(parentFrame, false, "CustomDialog Search?");  
            System.err.println("User chose yes.");  
            answer = true;  
            myDialog.getAnswer();  
            System.out.println("myDialog.getAnswer()="+myDialog.getAnswer());  
            myDialog.show();  

            if(myDialog.getAnswer()==true)  
            {  
                System.out.println("tofront");  
                this.toFront();  
            }  
            //setVisible(false);  
        }  
        else if(noButton == e.getSource()) {  
            System.err.println("User chose no.");  
            answer = false;  
            setVisible(false);  
        }  
    }  

}

import javax.swing.JDialog;   
import java.awt.event.ActionListener;  
import javax.swing.JPanel;  
import javax.swing.JFrame;  
import javax.swing.JLabel;  
import javax.swing.JButton;  
import java.awt.event.ActionEvent;  

public class CustomDialogSearch extends JDialog implements ActionListener {  
    private JPanel myPanel = null;  
    private JButton yesButton = null;  
    private JButton noButton = null;  
    private boolean answer = false;  
    public boolean getAnswer() { return answer; }  

    public CustomDialogSearch(JFrame frame, boolean modal, String myMessage) {  
        super(frame, modal);  
        myPanel = new JPanel();  
        getContentPane().add(myPanel);  
        myPanel.add(new JLabel(myMessage));  
        yesButton = new JButton("Yes");  
        yesButton.addActionListener(this);  
        myPanel.add(yesButton);   
        noButton = new JButton("No");  
        noButton.addActionListener(this);  
        myPanel.add(noButton);    
        pack();  
        setLocationRelativeTo(frame);  
        setVisible(true);  
    }  

    public void actionPerformed(ActionEvent e) {  
        if(yesButton == e.getSource()) {  
            System.err.println("Search User chose yes.");  
            answer = true;  
            //setVisible(false);  
        }  
        else if(noButton == e.getSource()) {  
            System.err.println("Search User chose no.");  
            answer = false;  
            setVisible(false);  
        }  
    }  

}

I even tried to set CustomDialogSearch's parent to be CustomDialog the behaviour still not correct.

I think you're in the right track here but you need to play with dialogs modality type. For instance:

Set the modality type of Custom Dialog Main ("parent" dialog) as Dialog.ModalityType.APPLICATION_MODAL. By doing this when this dialog is visible it will block all windows except its children.
Set the modality type of Custom Dialog Search ("child" dialog) as Dialog.ModalityType.MODELESS. This way it won't block any other window and you can go from the child to the parent and vice versa.

For a better understanding take a look to How to Use Modality in Dialogs article.

Example

Here is a code example about using modality as I've suggested above:

import java.awt.Dialog;
import java.awt.Window;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import javax.swing.JButton;
import javax.swing.JDialog;
import javax.swing.JFrame;
import javax.swing.SwingUtilities;

public class Demo {

    private void createAndShowGUI() {
        JButton button = new JButton("Create Parent modal dialog");
        button.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                JButton button = (JButton)e.getSource();
                JFrame owner = (JFrame)SwingUtilities.windowForComponent(button);
                Demo.this.createAndShowParentDialog(owner);                
            }
        });

        JFrame frame = new JFrame("Demo");
        frame.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE);
        frame.getContentPane().add(button);
        frame.pack();
        frame.setLocationRelativeTo(null);
        frame.setVisible(true);
    }

    private void createAndShowParentDialog(JFrame owner) {
        JButton button = new JButton("Create Child non-modal dialog");
        button.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                JButton button = (JButton)e.getSource();
                JDialog parent = (JDialog)SwingUtilities.windowForComponent(button);
                Demo.this.createAndShowChildrenDialog(parent);                
            }
        });

        JDialog parentDialog = new JDialog(owner, "Parent dialog");
        parentDialog.setModalityType(Dialog.ModalityType.APPLICATION_MODAL);
        parentDialog.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE);
        parentDialog.getContentPane().add(button);
        parentDialog.pack();
        parentDialog.setLocationRelativeTo(null);        
        parentDialog.setVisible(true);
    }

    private void createAndShowChildrenDialog(JDialog parent) {        
        JButton backButton = new JButton("Back to parent dialog");
        backButton.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                JButton button = (JButton)e.getSource();
                Window dialog = SwingUtilities.windowForComponent(button);
                dialog.getOwner().toFront();
            }
        });

        JDialog childDialog = new JDialog(parent, "Child dialog");
        childDialog.setModalityType(Dialog.ModalityType.MODELESS);
        childDialog.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE);
        childDialog.getContentPane().add(backButton);
        childDialog.pack();
        childDialog.setLocationRelativeTo(null);        
        childDialog.setVisible(true);
    }


    public static void main(String[] args) {
        SwingUtilities.invokeLater(new Runnable() {
            @Override
            public void run() {
                new Demo().createAndShowGUI();
            }
        });
    }    
}

Edit

I can select the windows of parent and child JDialogs but when I select the parent JDialog window, the child JDialog is still in front of parent JDialog.

Well I have a better understanding now on what is the problem. This behaviour depends on how native windowing system handles focused and active windows. Having said this if you call for instance toFront() it will attempt to place the window at the top of the stack BUT some platforms do not allow windows which own other windows to appear on top of its childre. The same happens when you call toBack() method. See the javadocs for more details.

I've tested my code on Windows 7 (32 bits if it makes any difference) and parent dialog becomes focused but its children still showing (not focused) at the top. As mentioned above it's up to the windowing system decide how to handle this matter.