Gdk / X11 Screen Capture

Question

I want to write a program that continuously captures the screen and does some modifications to the image. A complete test program can be found at:

https://gist.github.com/blogsh/eb4dd4b96aca468c8bfa

However, I ran into some problems. The first experiment I did was using the Gdk root window, create a Cairo context from it and then using it's target as the source for another window, where the contents are painted to:

mScreenContext = Gdk::Screen::get_default()->get_root_window()->create_cairo_context()
...
context->set_source(mScreenContext->get_target(), 0, 0);
context->paint();

This works perfectly fine (variant 1 in the source above). It just draws the whole screen into another window. So my next step was to try to save the contents into a Cairo ImageSurface in order to modify it:

mImageContext->set_source(mScreenContext->get_target(), 0, 0);
mImageContext->paint();

context->set_source(mImageSurface, 0, 0);
context->paint();

The surprising thing is, that for the first drawing of the Gtk window the screen is captured and drawn. Unfortunately nothing is happening afterwards, still the initial screen is displayed. How can this behaviour explained? I must admit I don't know much about the underlying processes here, so maybe someone can give some hints?

A third variant using Gdk::Pixbuf yields the exact same behaviour:

mScreenBuffer = Gdk::Pixbuf::create(mGdkRootWindow, 0, 0, mScreenWidth, mScreenHeight);
Gdk::Cairo::set_source_pixbuf(context, mScreenBuffer, 0, 0);
context->paint();

Finally (variant 4) I tried using X11 directly:

Display *display = XOpenDisplay((char*)0);
XImage *image = XGetImage(display, RootWindow(display, DefaultScreen(display)), 0, 0, mScreenWidth, mScreenHeight, AllPlanes, XYPixmap);

mScreenBuffer = Gdk::Pixbuf::create_from_data((const guint8*)image->data, Gdk::COLORSPACE_RGB, 0, 8, mScreenWidth, mScreenHeight, mScreenWidth);
Gdk::Cairo::set_source_pixbuf(context, mScreenBuffer, 0, 0);
context->paint();

XFree(image);

Actually, this works (although I didn't make any efforts to match the pixel format correctly yet), but it is awfully slow!

So I would appreciate any hints on what the issue with the two Gdk variants is and/or how to speed up the X11 approach. Or maybe someone knows a completely different approach to capture the screen in a fast way.

Unfortunately I'm not so familiar with that whole topic, but another idea would be to use a OpenGL-based window manager, where I could read the framebuffer directly? Does that make sense?

The main idea of the program is that I have a projector which I cannot position directly in front of a wall. So my idea is to capture the screen, do some bilinear transformation to account for the skwedness of the projection and then display the modified screen in another window, which will be shown on the projector...

Answer 1

XShmGetImage and XShmPutImage are faster than XGetImage and XPutImage. In the next example, I create two images: src and dst. In each iteration I save a screenshot in src, then I render a scaled version of it in dst.

The image below shows the example running in the window entitled "screencap". At low demand, it runs at 60 fps (as seen in the terminal at the top-right corner). At high demand, performance can drop to 25fps.

Test computer:

Display resolution: 1920x1080
Graphic card: ATI Radeon HD 4200 (integrated)
CPU: AMD Phenom(tm) II X4 945, 3013.85 MHz
Window manager: XFCE 4.12 (compositing off)
Operating system: OpenBSD 5.9
Tested also in Linux (openSUSE Leap 42.1)

#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <math.h>
#include <stdbool.h>
#include <sys/shm.h>
#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <X11/extensions/XShm.h>
#ifdef __linux__
    #include <sys/time.h>
#endif

// comment the next line to busy-wait at each frame
//#define __SLEEP__
#define FRAME  16667
#define PERIOD 1000000
#define NAME   "screencap"
#define NAMESP "         "
#define BPP    4

struct shmimage
{
    XShmSegmentInfo shminfo ;
    XImage * ximage ;
    unsigned int * data ; // will point to the image's BGRA packed pixels
} ;

void initimage( struct shmimage * image )
{
    image->ximage = NULL ;
    image->shminfo.shmaddr = (char *) -1 ;
}

void destroyimage( Display * dsp, struct shmimage * image )
{
    if( image->ximage )
    {
        XShmDetach( dsp, &image->shminfo ) ;
        XDestroyImage( image->ximage ) ;
        image->ximage = NULL ;
    }

    if( image->shminfo.shmaddr != ( char * ) -1 )
    {
        shmdt( image->shminfo.shmaddr ) ;
        image->shminfo.shmaddr = ( char * ) -1 ;
    }
}

int createimage( Display * dsp, struct shmimage * image, int width, int height )
{
    // Create a shared memory area 
    image->shminfo.shmid = shmget( IPC_PRIVATE, width * height * BPP, IPC_CREAT | 0600 ) ;
    if( image->shminfo.shmid == -1 )
    {
        perror( NAME ) ;
        return false ;
    }

    // Map the shared memory segment into the address space of this process
    image->shminfo.shmaddr = (char *) shmat( image->shminfo.shmid, 0, 0 ) ;
    if( image->shminfo.shmaddr == (char *) -1 )
    {
        perror( NAME ) ;
        return false ;
    }

    image->data = (unsigned int*) image->shminfo.shmaddr ;
    image->shminfo.readOnly = false ;

    // Mark the shared memory segment for removal
    // It will be removed even if this program crashes
    shmctl( image->shminfo.shmid, IPC_RMID, 0 ) ;

    // Allocate the memory needed for the XImage structure
    image->ximage = XShmCreateImage( dsp, XDefaultVisual( dsp, XDefaultScreen( dsp ) ),
                        DefaultDepth( dsp, XDefaultScreen( dsp ) ), ZPixmap, 0,
                        &image->shminfo, 0, 0 ) ;
    if( !image->ximage )
    {
        destroyimage( dsp, image ) ;
        printf( NAME ": could not allocate the XImage structure\n" ) ;
        return false ;
    }

    image->ximage->data = (char *)image->data ;
    image->ximage->width = width ;
    image->ximage->height = height ;

    // Ask the X server to attach the shared memory segment and sync
    XShmAttach( dsp, &image->shminfo ) ;
    XSync( dsp, false ) ;
    return true ;
}

void getrootwindow( Display * dsp, struct shmimage * image )
{
    XShmGetImage( dsp, XDefaultRootWindow( dsp ), image->ximage, 0, 0, AllPlanes ) ;
}

long timestamp( )
{
   struct timeval tv ;
   struct timezone tz ;
   gettimeofday( &tv, &tz ) ;
   return tv.tv_sec*1000000L + tv.tv_usec ;
}

Window createwindow( Display * dsp, int width, int height )
{
    unsigned long mask = CWBackingStore ;
    XSetWindowAttributes attributes ;
    attributes.backing_store = NotUseful ;
    mask |= CWBackingStore ;
    Window window = XCreateWindow( dsp, DefaultRootWindow( dsp ),
            0, 0, width, height, 0,
            DefaultDepth( dsp, XDefaultScreen( dsp ) ),
            InputOutput, CopyFromParent, mask, &attributes ) ;
    XStoreName( dsp, window, NAME );
    XSelectInput( dsp, window, StructureNotifyMask ) ;
    XMapWindow( dsp, window );
    return window ;
}

void destroywindow( Display * dsp, Window window )
{
    XDestroyWindow( dsp, window );
}

unsigned int getpixel( struct shmimage * src, struct shmimage * dst,
                       int j, int i, int w, int h )
{
    int x = (float)(i * src->ximage->width) / (float)w ;
    int y = (float)(j * src->ximage->height) / (float)h ;
    return src->data[ y * src->ximage->width + x ] ;
}

int processimage( struct shmimage * src, struct shmimage * dst )
{
    int sw = src->ximage->width ;
    int sh = src->ximage->height ;
    int dw = dst->ximage->width ;
    int dh = dst->ximage->height ;

    // Here you can set the resulting position and size of the captured screen
    // Because of the limitations of this example, it must fit in dst->ximage
    int w = dw / 2 ;
    int h = dh / 2 ;
    int x = ( dw - w ) ;
    int y = ( dh - h ) / 2 ;

    // Just in case...
    if( x < 0 || y < 0 || x + w > dw || y + h > dh || sw < dw || sh < dh )
    {
        printf( NAME   ": This is only a limited example\n" ) ;
        printf( NAMESP "  Please implement a complete scaling algorithm\n" ) ;
        return false ;
    }

    unsigned int * d = dst->data + y * dw + x ;
    int r = dw - w ;
    int j, i ;
    for( j = 0 ; j < h ; ++j )
    {
        for( i = 0 ; i < w ; ++i )
        {
            *d++ = getpixel( src, dst, j, i, w, h ) ;
        }
        d += r ;
    }
    return true ;
}

int run( Display * dsp, Window window, struct shmimage * src, struct shmimage * dst )
{
    XGCValues xgcvalues ;
    xgcvalues.graphics_exposures = False ;
    GC gc = XCreateGC( dsp, window, GCGraphicsExposures, &xgcvalues ) ;

    Atom delete_atom = XInternAtom( dsp, "WM_DELETE_WINDOW", False ) ;
    XSetWMProtocols( dsp, window, &delete_atom, True ) ;

    XEvent xevent ;
    int running = true ;
    int initialized = false ;
    int dstwidth = dst->ximage->width ;
    int dstheight = dst->ximage->height ;
    long framets = timestamp( ) ;
    long periodts = timestamp( ) ;
    long frames = 0 ;
    int fd = ConnectionNumber( dsp ) ;
    while( running )
    {
        while( XPending( dsp ) )
        {
            XNextEvent( dsp, &xevent ) ;
            if( ( xevent.type == ClientMessage && xevent.xclient.data.l[0] == delete_atom )
                || xevent.type == DestroyNotify )
            {
                running = false ;
                break ;
            }
            else if( xevent.type == ConfigureNotify )
            {
                if( xevent.xconfigure.width == dstwidth
                    && xevent.xconfigure.height == dstheight )
                {
                    initialized = true ;
                }
            }
        }
        if( initialized )
        {
            getrootwindow( dsp, src ) ;
            if( !processimage( src, dst ) )
            {
                return false ;
            }
            XShmPutImage( dsp, window, gc, dst->ximage,
                          0, 0, 0, 0, dstwidth, dstheight, False ) ;
            XSync( dsp, False ) ;

            int frameus = timestamp( ) - framets ;
            ++frames ;
            while( frameus < FRAME )
            {
                #if defined( __SLEEP__ )
                usleep( FRAME - frameus ) ;
                #endif
                frameus = timestamp( ) - framets ;
            }
            framets = timestamp( ) ;

            int periodus = timestamp( ) - periodts ;
            if( periodus >= PERIOD )
            {
                printf( "fps: %d\n", (int)round( 1000000.0L * frames / periodus ) ) ;
                frames = 0 ;
                periodts = framets ;
            }
        }
    }
    return true ;
}

int main( int argc, char * argv[] )
{
    Display * dsp = XOpenDisplay( NULL ) ;
    if( !dsp )
    {
        printf( NAME ": could not open a connection to the X server\n" ) ;
        return 1 ;
    }

    if( !XShmQueryExtension( dsp ) )
    {
        XCloseDisplay( dsp ) ;
        printf( NAME ": the X server does not support the XSHM extension\n" ) ;
        return 1 ;
    }

    int screen = XDefaultScreen( dsp ) ;
    struct shmimage src, dst ;
    initimage( &src ) ;
    int width = XDisplayWidth( dsp, screen ) ;
    int height = XDisplayHeight( dsp, screen ) ;
    if( !createimage( dsp, &src, width, height ) )
    {
        XCloseDisplay( dsp ) ;
        return 1 ;
    }
    initimage( &dst ) ;
    int dstwidth = width / 2 ;
    int dstheight = height / 2 ;
    if( !createimage( dsp, &dst, dstwidth, dstheight ) )
    {
        destroyimage( dsp, &src ) ;
        XCloseDisplay( dsp ) ;
        return 1 ;
    }

    if( dst.ximage->bits_per_pixel != 32 )
    {
        destroyimage( dsp, &src ) ;
        destroyimage( dsp, &dst ) ;
        XCloseDisplay( dsp ) ;
        printf( NAME   ": This is only a limited example\n" ) ;
        printf( NAMESP "  Please add support for all pixel formats using: \n" ) ;
        printf( NAMESP "      dst.ximage->bits_per_pixel\n" ) ;
        printf( NAMESP "      dst.ximage->red_mask\n" ) ;
        printf( NAMESP "      dst.ximage->green_mask\n" ) ;
        printf( NAMESP "      dst.ximage->blue_mask\n" ) ;
        return 1 ;
    }

    Window window = createwindow( dsp, dstwidth, dstheight ) ;
    run( dsp, window, &src, &dst ) ;
    destroywindow( dsp, window ) ;  

    destroyimage( dsp, &src ) ;
    destroyimage( dsp, &dst ) ;
    XCloseDisplay( dsp ) ;
    return 0 ;
}

This is just an example. If you like how it performs, you should consider adding a better appropriate scaling algorithm and support for all pixels formats.

You can compile the example like this:

gcc screencap.c -o screencap -std=c99 -I/usr/X11R6/include -L/usr/X11R6/lib -lX11 -lXext -lm

Answer 2

Call cairo_surface_mark_dirty() on the cairo surface that refers to the root window. Since you seem to only have a cairo context and not a surface directly: You can use cairo_surface_get_target() to get the surface from the cairo context.

Cairo assumes that you only draw to the surface through the cairo API, unless you tell it that you changed something outside of cairo. Downloading a screenshot is expensive and so cairo caches the result so that it can re-use it later.

Likely, you will also have to call cairo_surface_flush() before the call to cairo_surface_mark_dirty()...