Intro

Here is an example to illustrate the problem. Consider I am tracking and displaying mouse global current position and last click button and position to the user. Here is an image:

To archive capturing click events on windows box, that would and will be sent to the other programs event messaging queue, I create a hook using winapi namely user32.dll library. This is outside JDK sandbox, so I use JNA to call the native library.

This all works perfectly, but it does not close as I expect it to.

My question is - How do I properly close following example program?

Example source

Code below is not fully written by Me, but taken from this question in Oracle forum and partly fixed.

import java.awt.AWTException;
import java.awt.Dimension;
import java.awt.EventQueue;
import java.awt.GridLayout;
import java.awt.MouseInfo;
import java.awt.Point;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;

import javax.swing.JFrame;
import javax.swing.JLabel;

import com.sun.jna.Native;
import com.sun.jna.NativeLong;
import com.sun.jna.Platform;
import com.sun.jna.Structure;
import com.sun.jna.platform.win32.BaseTSD.ULONG_PTR;
import com.sun.jna.platform.win32.Kernel32;
import com.sun.jna.platform.win32.User32;
import com.sun.jna.platform.win32.WinDef.HWND;
import com.sun.jna.platform.win32.WinDef.LRESULT;
import com.sun.jna.platform.win32.WinDef.WPARAM;
import com.sun.jna.platform.win32.WinUser.HHOOK;
import com.sun.jna.platform.win32.WinUser.HOOKPROC;
import com.sun.jna.platform.win32.WinUser.MSG;
import com.sun.jna.platform.win32.WinUser.POINT;

public class MouseExample {
    final JFrame jf;
    final JLabel jl1, jl2;
    final CWMouseHook mh;
    final Ticker jt;

    public class Ticker extends Thread {
        public boolean update = true;

        public void done() {
            update = false;
        }

        public void run() {
            try {
                Point p, l = MouseInfo.getPointerInfo().getLocation();
                int i = 0;
                while (update == true) {
                    try {
                        p = MouseInfo.getPointerInfo().getLocation();
                        if (!p.equals(l)) {
                            l = p;
                            jl1.setText(new GlobalMouseClick(p.x, p.y)
                                    .toString());
                        }

                        Thread.sleep(35);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                        return;
                    }
                }
            } catch (Exception e) {
                update = false;
            }
        }
    }

    public MouseExample() throws AWTException, UnsupportedOperationException {
        this.jl1 = new JLabel("{}");
        this.jl2 = new JLabel("{}");
        this.jf = new JFrame();
        this.jt = new Ticker();
        this.jt.start();
        this.mh = new CWMouseHook() {
            @Override
            public void globalClickEvent(GlobalMouseClick m) {
                jl2.setText(m.toString());
            }
        };

        mh.setMouseHook();

        jf.setLayout(new GridLayout(2, 2));
        jf.add(new JLabel("Position"));
        jf.add(jl1);
        jf.add(new JLabel("Last click"));
        jf.add(jl2);
        jf.addWindowListener(new WindowAdapter() {
            public void windowClosing(WindowEvent we) {
                mh.dispose();
                jt.done();
                jf.dispose();
            }
        });
        jf.setLocation(new Point(0, 0));
        jf.setPreferredSize(new Dimension(200, 90));
        jf.pack();
        jf.setVisible(true);
    }

    public static class GlobalMouseClick {
        private char c;
        private int x, y;

        public GlobalMouseClick(char c, int x, int y) {
            super();
            this.c = c;
            this.x = x;
            this.y = y;
        }

        public GlobalMouseClick(int x, int y) {
            super();
            this.x = x;
            this.y = y;
        }

        public char getC() {
            return c;
        }

        public void setC(char c) {
            this.c = c;
        }

        public int getX() {
            return x;
        }

        public void setX(int x) {
            this.x = x;
        }

        public int getY() {
            return y;
        }

        public void setY(int y) {
            this.y = y;
        }

        @Override
        public String toString() {
            return (c != 0 ? c : "") + " [" + x + "," + y + "]";
        }
    }

    public static class CWMouseHook {
        public User32 USER32INST;

        public CWMouseHook() throws UnsupportedOperationException {
            if (!Platform.isWindows()) {
                throw new UnsupportedOperationException(
                        "Not supported on this platform.");
            }
            USER32INST = User32.INSTANCE;
            mouseHook = hookTheMouse();
            Native.setProtected(true);
        }

        private static LowLevelMouseProc mouseHook;
        private HHOOK hhk;
        private boolean isHooked = false;

        public static final int WM_LBUTTONDOWN = 513;
        public static final int WM_LBUTTONUP = 514;
        public static final int WM_RBUTTONDOWN = 516;
        public static final int WM_RBUTTONUP = 517;
        public static final int WM_MBUTTONDOWN = 519;
        public static final int WM_MBUTTONUP = 520;

        public void dispose() {
            unsetMouseHook();
            mousehook_thread = null;
            mouseHook = null;
            hhk = null;
            USER32INST = null;
        }

        public void unsetMouseHook() {
            isHooked = false;
            USER32INST.UnhookWindowsHookEx(hhk);
            System.out.println("Mouse hook is unset.");
        }

        public boolean isIsHooked() {
            return isHooked;
        }

        public void globalClickEvent(GlobalMouseClick m) {
            System.out.println(m);
        }

        private Thread mousehook_thread;

public void setMouseHook() {
    mousehook_thread = new Thread(new Runnable() {
        @Override
        public void run() {
            try {
                if (!isHooked) {
                    hhk = USER32INST.SetWindowsHookEx(14, mouseHook,
                            Kernel32.INSTANCE.GetModuleHandle(null), 0);

                    isHooked = true;

                    System.out
                            .println("Mouse hook is set. Click anywhere.");
                    // message dispatch loop (message pump)
                    MSG msg = new MSG();
                    while ((USER32INST.GetMessage(msg, null, 0, 0)) != 0) {
                        USER32INST.TranslateMessage(msg);
                        USER32INST.DispatchMessage(msg);
                        if (!isHooked)
                            break;
                    }
                } else
                    System.out
                            .println("The Hook is already installed.");
            } catch (Exception e) {
                System.err.println("Caught exception in MouseHook!");
            }
        }
    });
    mousehook_thread.start();
}
        private interface LowLevelMouseProc extends HOOKPROC {
            LRESULT callback(int nCode, WPARAM wParam, MOUSEHOOKSTRUCT lParam);
        }

        private LowLevelMouseProc hookTheMouse() {
            return new LowLevelMouseProc() {
                @Override
                public LRESULT callback(int nCode, WPARAM wParam,
                        MOUSEHOOKSTRUCT info) {
                    if (nCode >= 0) {
                        switch (wParam.intValue()) {
                        case CWMouseHook.WM_LBUTTONDOWN:
                            globalClickEvent(new GlobalMouseClick('L',
                                    info.pt.x, info.pt.y));
                            break;
                        case CWMouseHook.WM_RBUTTONDOWN:
                            globalClickEvent(new GlobalMouseClick('R',
                                    info.pt.x, info.pt.y));
                            break;
                        case CWMouseHook.WM_MBUTTONDOWN:
                            globalClickEvent(new GlobalMouseClick('M',
                                    info.pt.x, info.pt.y));
                            break;
                        default:
                            break;
                        }
                    }
                    return USER32INST.CallNextHookEx(hhk, nCode, wParam,
                            info.getPointer());
                }
            };
        }

        public class Point extends Structure {
            public class ByReference extends Point implements
                    Structure.ByReference {
            };

            public NativeLong x;
            public NativeLong y;
        }

        public static class MOUSEHOOKSTRUCT extends Structure {
            public static class ByReference extends MOUSEHOOKSTRUCT implements
                    Structure.ByReference {
            };

            public POINT pt;
            public HWND hwnd;
            public int wHitTestCode;
            public ULONG_PTR dwExtraInfo;
        }
    }

    public static void main(String[] args) {
        EventQueue.invokeLater(new Runnable() {
            @Override
            public void run() {
                try {
                    new MouseExample();
                } catch (AWTException e) {
                    e.printStackTrace();
                }
            }
        });
    }
}

In your thread somewehere you should call

User32.PostQuitMessage(0)

to notify the native thread (hook) that you no longer need it. When you do that, in your code the check

while ((USER32INST.GetMessage(msg, null, 0, 0)) != 0)

sees that you no longer need the hook and terminates it on the native side. I suggest you first try closing the hook with an internal for it event, e.g