How to implement alt+MMB camera rotation like in 3

PREREQUISITES

Let me start the question by providing some boilerplate code we'll use to play around:

mcve_framework.py:

from OpenGL.GL import *
from OpenGL.GLU import *
from OpenGL.GLUT import *

import glm
from glm import cross, normalize, unProject, vec2, vec3, vec4


# -------- Camera --------
class BaseCamera():

    def __init__(
        self,
        eye=None, target=None, up=None,
        fov=None, near=0.1, far=100000,
        delta_zoom=10
    ):
        self.eye = eye or glm.vec3(0, 0, 1)
        self.target = target or glm.vec3(0, 0, 0)
        self.up = up or glm.vec3(0, 1, 0)
        self.original_up = glm.vec3(self.up)
        self.fov = fov or glm.radians(45)
        self.near = near
        self.far = far
        self.delta_zoom = delta_zoom

    def update(self, aspect):
        self.view = glm.lookAt(
            self.eye, self.target, self.up
        )
        self.projection = glm.perspective(
            self.fov, aspect, self.near, self.far
        )

    def move(self, dx, dy, dz, dt):
        if dt == 0:
            return

        forward = normalize(self.target - self.eye) * dt
        right = normalize(cross(forward, self.up)) * dt
        up = self.up * dt

        offset = right * dx
        self.eye += offset
        self.target += offset

        offset = up * dy
        self.eye += offset
        self.target += offset

        offset = forward * dz
        self.eye += offset
        self.target += offset

    def zoom(self, *args):
        x = args[2]
        y = args[3]
        v = glGetIntegerv(GL_VIEWPORT)
        viewport = vec4(float(v[0]), float(v[1]), float(v[2]), float(v[3]))
        height = viewport.w

        pt_wnd = vec3(x, height - y, 1.0)
        pt_world = unProject(pt_wnd, self.view, self.projection, viewport)
        ray_cursor = glm.normalize(pt_world - self.eye)

        delta = args[1] * self.delta_zoom
        self.eye = self.eye + ray_cursor * delta
        self.target = self.target + ray_cursor * delta

    def load_projection(self):
        width = glutGet(GLUT_WINDOW_WIDTH)
        height = glutGet(GLUT_WINDOW_HEIGHT)

        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        gluPerspective(glm.degrees(self.fov), width / height, self.near, self.far)

    def load_modelview(self):
        e = self.eye
        t = self.target
        u = self.up

        glMatrixMode(GL_MODELVIEW)
        glLoadIdentity()
        gluLookAt(e.x, e.y, e.z, t.x, t.y, t.z, u.x, u.y, u.z)


class GlutController():

    FPS = 0
    ORBIT = 1
    PAN = 2

    def __init__(self, camera, velocity=100, velocity_wheel=100):
        self.velocity = velocity
        self.velocity_wheel = velocity_wheel
        self.camera = camera

    def glut_mouse(self, button, state, x, y):
        self.mouse_last_pos = vec2(x, y)
        self.mouse_down_pos = vec2(x, y)

        if button == GLUT_LEFT_BUTTON:
            self.mode = self.FPS
        elif button == GLUT_RIGHT_BUTTON:
            self.mode = self.ORBIT
        else:
            self.mode = self.PAN

    def glut_motion(self, x, y):
        pos = vec2(x, y)
        move = self.mouse_last_pos - pos
        self.mouse_last_pos = pos

        if self.mode == self.FPS:
            self.camera.rotate_target(move * 0.005)
        elif self.mode == self.ORBIT:
            self.camera.rotate_around_origin(move * 0.005)

    def glut_mouse_wheel(self, *args):
        self.camera.zoom(*args)

    def process_inputs(self, keys, dt):
        dt *= 10 if keys[' '] else 1
        ammount = self.velocity * dt

        if keys['w']:
            self.camera.move(0, 0, 1, ammount)
        if keys['s']:
            self.camera.move(0, 0, -1, ammount)
        if keys['d']:
            self.camera.move(1, 0, 0, ammount)
        if keys['a']:
            self.camera.move(-1, 0, 0, ammount)
        if keys['q']:
            self.camera.move(0, -1, 0, ammount)
        if keys['e']:
            self.camera.move(0, 1, 0, ammount)
        if keys['+']:
            self.camera.fov += radians(ammount)
        if keys['-']:
            self.camera.fov -= radians(ammount)


# -------- Mcve --------
class BaseWindow:

    def __init__(self, w, h, camera):
        self.width = w
        self.height = h

        glutInit()
        glutSetOption(GLUT_MULTISAMPLE, 16)
        glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGB | GLUT_DEPTH | GLUT_MULTISAMPLE)
        glutInitWindowSize(w, h)
        glutCreateWindow('OpenGL Window')

        self.keys = {chr(i): False for i in range(256)}

        self.startup()

        glutReshapeFunc(self.reshape)
        glutDisplayFunc(self.display)
        glutMouseFunc(self.controller.glut_mouse)
        glutMotionFunc(self.controller.glut_motion)
        glutMouseWheelFunc(self.controller.glut_mouse_wheel)
        glutKeyboardFunc(self.keyboard_func)
        glutKeyboardUpFunc(self.keyboard_up_func)
        glutIdleFunc(self.idle_func)

    def keyboard_func(self, *args):
        try:
            key = args[0].decode("utf8")

            if key == "\x1b":
                glutLeaveMainLoop()

            self.keys[key] = True
        except Exception as e:
            import traceback
            traceback.print_exc()

    def keyboard_up_func(self, *args):
        try:
            key = args[0].decode("utf8")
            self.keys[key] = False
        except Exception as e:
            pass

    def startup(self):
        raise NotImplementedError

    def display(self):
        raise NotImplementedError

    def run(self):
        glutMainLoop()

    def idle_func(self):
        glutPostRedisplay()

    def reshape(self, w, h):
        glViewport(0, 0, w, h)
        self.width = w
        self.height = h

In case you want to use the above code you'll just need to install pyopengl and pygml. After that, you can just create your own BaseWindow subclass, override startup and render and you should have a very basic glut window with simple functionality such as camera rotation/zooming as well as some methods to render points/triangles/quads and indexed_triangles/indexed_quads.

WHAT'S DONE

mcve_camera_arcball.py

import time

from OpenGL.GL import *
from OpenGL.GLU import *
from OpenGL.GLUT import *

import glm
from mcve_framework import BaseCamera, BaseWindow, GlutController


def line(p0, p1, color=None):
    c = color or glm.vec3(1, 1, 1)
    glColor3f(c.x, c.y, c.z)
    glVertex3f(p0.x, p0.y, p0.z)
    glVertex3f(p1.x, p1.y, p1.z)


def grid(segment_count=10, spacing=1, yup=True):
    size = segment_count * spacing
    right = glm.vec3(1, 0, 0)
    forward = glm.vec3(0, 0, 1) if yup else glm.vec3(0, 1, 0)
    x_axis = right * size
    z_axis = forward * size

    i = -segment_count

    glBegin(GL_LINES)
    while i <= segment_count:
        p0 = -x_axis + forward * i * spacing
        p1 = x_axis + forward * i * spacing
        line(p0, p1)
        p0 = -z_axis + right * i * spacing
        p1 = z_axis + right * i * spacing
        line(p0, p1)
        i += 1
    glEnd()


def axis(size=1.0, yup=True):
    right = glm.vec3(1, 0, 0)
    forward = glm.vec3(0, 0, 1) if yup else glm.vec3(0, 1, 0)
    x_axis = right * size
    z_axis = forward * size
    y_axis = glm.cross(forward, right) * size
    glBegin(GL_LINES)
    line(x_axis, glm.vec3(0, 0, 0), glm.vec3(1, 0, 0))
    line(y_axis, glm.vec3(0, 0, 0), glm.vec3(0, 1, 0))
    line(z_axis, glm.vec3(0, 0, 0), glm.vec3(0, 0, 1))
    glEnd()


class Camera(BaseCamera):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def rotate_target(self, delta):
        right = glm.normalize(glm.cross(self.target - self.eye, self.up))
        M = glm.mat4(1)
        M = glm.translate(M, self.eye)
        M = glm.rotate(M, delta.y, right)
        M = glm.rotate(M, delta.x, self.up)
        M = glm.translate(M, -self.eye)
        self.target = glm.vec3(M * glm.vec4(self.target, 1.0))

    def rotate_around_target(self, target, delta):
        right = glm.normalize(glm.cross(self.target - self.eye, self.up))
        ammount = (right * delta.y + self.up * delta.x)
        M = glm.mat4(1)
        M = glm.rotate(M, ammount.z, glm.vec3(0, 0, 1))
        M = glm.rotate(M, ammount.y, glm.vec3(0, 1, 0))
        M = glm.rotate(M, ammount.x, glm.vec3(1, 0, 0))
        self.eye = glm.vec3(M * glm.vec4(self.eye, 1.0))
        self.target = target
        self.up = self.original_up

    def rotate_around_origin(self, delta):
        return self.rotate_around_target(glm.vec3(0), delta)


class McveCamera(BaseWindow):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def startup(self):
        glEnable(GL_DEPTH_TEST)

        self.start_time = time.time()
        self.camera = Camera(
            eye=glm.vec3(200, 200, 200),
            target=glm.vec3(0, 0, 0),
            up=glm.vec3(0, 1, 0),
            delta_zoom=30
        )
        self.model = glm.mat4(1)
        self.controller = GlutController(self.camera)
        glPolygonMode(GL_FRONT_AND_BACK, GL_LINE)

    def display(self):
        self.controller.process_inputs(self.keys, 0.005)
        self.camera.update(self.width / self.height)

        glClearColor(0.2, 0.3, 0.3, 1.0)
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

        self.camera.load_projection()
        self.camera.load_modelview()

        glLineWidth(5)
        axis(size=70, yup=True)
        glLineWidth(1)
        grid(segment_count=7, spacing=10, yup=True)

        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        glOrtho(-1, 1, -1, 1, -1, 1)
        glMatrixMode(GL_MODELVIEW)
        glLoadIdentity()

        glutSwapBuffers()


if __name__ == '__main__':
    window = McveCamera(800, 600, Camera())
    window.run()

TODO

The end goal here is to figure out how to emulate the rotation used by 3dsmax when pressing Alt+MMB.

Right now with the current code you can move around using WASDQE keys (shift to accelerate), left/right button to rotate camera around it's/scene's center or zooming by using mouse wheel. As you can see, offset values are hardcoded, just adjust them to run smoothly in your box (I know there are proper methods to make the camera kinetics vectors to be cpu independent, that's not the point of my question)

REFERENCES

Let's try to disect a little bit more how the camera behaves when pressing alt+MMB on 3dsmax2018.

1) Rottion at "home" (camera at home happens when you press the home button on the top right gizmo, that will place the camera position at a fixed location and target at (0,0,0)):

2) Panning and rotation:

3) Zooming/Panning and rotation:

4) User interface

QUESTION: So next would be adding the necessary bits to implement arcball rotation when pressing alt+MMB... I say arcball rotation cos I assume 3ds max uses that method behind the curtains but I'm not really sure that's the method used by max so first I'd like to know what are the exact maths used by 3ds max when pressing alt+MMB and then just add the necessary code to the Camera class to achieve that task

You have to apply a rotation matrix around the x and y axis to the view matrix. First apply the rotation matrix around the y axis (up vector) then the current view matrix and finally the rotation on the x axis:

view-matrix = rotate-X * view-matrix * rotate-Y

The rotation works exactly as in "3ds max", ecept the proper location of the origin of the rotation (the pivot - pivotWorld) has to be defined.

A plausible solution is, that the pivot is the camera target (self.target).
At the begin the target is (0, 0, 0) which is the origin of the world. To rotate around the origin of the world, is the expected behaviour as long the target of the view is the center of the world. If the view is pan, then the target is still in the center of the viewport, because it is moved in the same way as the point of view - self.eye ans self.target are moved "parallel". This causes the the scene, still appears to rotate around a point in the center of the view (the new target) and seems to be the exact same behaviour, as in "3ds max".

def rotate_around_target(self, target, delta):

    # get the view matrix
    view = glm.lookAt(self.eye, self.target, self.up)

    # pivot in world sapace and view space
    #pivotWorld = glm.vec3(0, 0, 0)
    pivotWorld = self.target

    pivotView = glm.vec3(view * glm.vec4(*pivotWorld, 1))  

    # rotation around the vies pace x axis
    rotViewX    = glm.rotate( glm.mat4(1), -delta.y, glm.vec3(1, 0, 0) )
    rotPivotViewX   = glm.translate(glm.mat4(1), pivotView) * rotViewX * glm.translate(glm.mat4(1), -pivotView)  

    # rotation around the world space up vector
    rotWorldUp  = glm.rotate( glm.mat4(1), -delta.x, glm.vec3(0, 1, 0) )
    rotPivotWorldUp = glm.translate(glm.mat4(1), pivotWorld) * rotWorldUp * glm.translate(glm.mat4(1), -pivotWorld)

    # update view matrix
    view = rotPivotViewX * view * rotPivotWorldUp

    # decode eye, target and up from view matrix
    C = glm.inverse(view)
    targetDist  = glm.length(self.target - self.eye)
    self.eye    = glm.vec3(C[3])
    self.target = self.eye - glm.vec3(C[2]) * targetDist 
    self.up     = glm.vec3(C[1])

But still there is an issue left. What is if the scene is zoomed?
In the current implementation the distance from the camera to the target point is kept constant. In case of zoom this may be incorrect, the direction from the point of view (self.eye) to the target (self.target) has to stay the same, but possibly the distance to the target has to be changed according to the zoom.
I suggest to do the following changes to the method zoom of the class BaseCamera:

class BaseCamera():

    def zoom(self, *args):
        x = args[2]
        y = args[3]
        v = glGetIntegerv(GL_VIEWPORT)
        viewport = vec4(float(v[0]), float(v[1]), float(v[2]), float(v[3]))
        height = viewport.w

        pt_wnd = vec3(x, height - y, 1.0)
        pt_world = unProject(pt_wnd, self.view, self.projection, viewport)
        ray_cursor = glm.normalize(pt_world - self.eye)

        # calculate the "zoom" vector
        delta       = args[1] * self.delta_zoom
        zoom_vec    = ray_cursor * delta

        # get the direction of sight and the distance to the target 
        sight_vec   = self.target - self.eye
        target_dist = glm.length(sight_vec)
        sight_vec   = sight_vec / target_dist

        # modify the distance to the target
        delta_dist = glm.dot(sight_vec, zoom_vec)
        if (target_dist - delta_dist) > 0.01: # the direction has to kept in any case
            target_dist -= delta_dist

        # update the eye postion and the target
        self.eye    = self.eye + zoom_vec
        self.target = self.eye + sight_vec * target_dist