27

I want to use OpenCV to process my desktop as if it were a video stream.
I am familiar with OpenCV.
I am not familiar with the Windows API. I realize there are other ways to capture the screen, but for the purposes of my question, I need it to be done using OpenCV.

Here is my (super naive) code:

HWND hDesktopWnd;
HDC hDesktopDC;
hDesktopWnd=GetDesktopWindow();
hDesktopDC=GetDC(hDesktopWnd);

// get the height and width of the screen
int height = GetSystemMetrics(SM_CYVIRTUALSCREEN);
int width = GetSystemMetrics(SM_CXVIRTUALSCREEN);

// create a bitmap
HBITMAP hbDesktop = CreateCompatibleBitmap( hDesktopDC, width, height);

Mat src(height,width,CV_8UC4);
src.data = (uchar*)hbDesktop;

imshow("output",src);  //fails :(

There are similar questions on StackOverflow, but they are either for the old-style OpenCV, or for Android operating system.
I'm on windows 7 64x
Opencv 2.4.3

Thanks anyone who can answer this question.

3

2 Answers 2

45

After MUCH trial and error, I managed to write a function to do it. here it is for anyone else who might want it:

#include "stdafx.h"
#include "opencv2/core/core.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include <opencv2/highgui/highgui.hpp>
#include <Windows.h>
#include <iostream>
#include <string>
using namespace std;
using namespace cv;

Mat hwnd2mat(HWND hwnd){

    HDC hwindowDC,hwindowCompatibleDC;

    int height,width,srcheight,srcwidth;
    HBITMAP hbwindow;
    Mat src;
    BITMAPINFOHEADER  bi;
    
    hwindowDC=GetDC(hwnd);
    hwindowCompatibleDC=CreateCompatibleDC(hwindowDC);
    SetStretchBltMode(hwindowCompatibleDC,COLORONCOLOR);  
    
    RECT windowsize;    // get the height and width of the screen
    GetClientRect(hwnd, &windowsize);

    srcheight = windowsize.bottom;
    srcwidth = windowsize.right;
    height = windowsize.bottom/2;  //change this to whatever size you want to resize to
    width = windowsize.right/2;
    
    src.create(height,width,CV_8UC4);

    // create a bitmap
    hbwindow = CreateCompatibleBitmap( hwindowDC, width, height);
    bi.biSize = sizeof(BITMAPINFOHEADER);    //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
    bi.biWidth = width;    
    bi.biHeight = -height;  //this is the line that makes it draw upside down or not
    bi.biPlanes = 1;    
    bi.biBitCount = 32;    
    bi.biCompression = BI_RGB;    
    bi.biSizeImage = 0;  
    bi.biXPelsPerMeter = 0;    
    bi.biYPelsPerMeter = 0;    
    bi.biClrUsed = 0;    
    bi.biClrImportant = 0;
    
    // use the previously created device context with the bitmap
    SelectObject(hwindowCompatibleDC, hbwindow);
    // copy from the window device context to the bitmap device context
    StretchBlt( hwindowCompatibleDC, 0,0, width, height, hwindowDC, 0, 0,srcwidth,srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
    GetDIBits(hwindowCompatibleDC,hbwindow,0,height,src.data,(BITMAPINFO *)&bi,DIB_RGB_COLORS);  //copy from hwindowCompatibleDC to hbwindow

    // avoid memory leak
    DeleteObject (hbwindow); DeleteDC(hwindowCompatibleDC); ReleaseDC(hwnd, hwindowDC);

    return src;
}
5
  • 3
    Following lines could be added to avoid memory leak: DeleteObject (hbwindow); DeleteDC(hwindowCompatibleDC); ReleaseDC(hwnd, hwindowDC);
    – Steck
    Jun 6, 2013 at 15:05
  • Great. Thanks for the code. I've added the code of @Steck to your post to make it complete.
    – Samuel
    Mar 20, 2014 at 20:51
  • Thanks for the code, it works perfectly. Added information of your initial question to get handle of screen. Added comment of @Steck to my code and edited your answer with those memory frees. Hope that was ok.
    – Micka
    Sep 22, 2014 at 16:07
  • I'm trying to use this to capture a MEmu window. It captures just the "frame" of the windows (window title, menu items, close button, etc). Any idea why this happens?
    – sthiago
    Dec 22, 2016 at 5:49
  • Does this work in Win10? With some adjustments in code about height and width, for me it returns black frame when hwnd is window handle though when it's NULL it does what it's supposed to, return whole screen.
    – Brackets
    Aug 24, 2017 at 18:13
13

A better way to do it is do it while allocating memory to the pixels only once. so the only copy done here is the one that made by BitBlt

enter image description here

int main()

{

    int x_size = 800, y_size = 600; // <-- Your res for the image





    HBITMAP hBitmap; // <-- The image represented by hBitmap

    Mat matBitmap; // <-- The image represented by mat





    // Initialize DCs

    HDC hdcSys = GetDC(NULL); // Get DC of the target capture..
    HDC hdcMem = CreateCompatibleDC(hdcSys); // Create compatible DC 






    void *ptrBitmapPixels; // <-- Pointer variable that will contain the potinter for the pixels









    // Create hBitmap with Pointer to the pixels of the Bitmap
    BITMAPINFO bi; HDC hdc;
    ZeroMemory(&bi, sizeof(BITMAPINFO));
    bi.bmiHeader.biSize = sizeof(BITMAPINFOHEADER);
    bi.bmiHeader.biWidth = x_size;
    bi.bmiHeader.biHeight = -y_size;  //negative so (0,0) is at top left

    bi.bmiHeader.biPlanes = 1;

    bi.bmiHeader.biBitCount = 32;
    hdc = GetDC(NULL);
    hBitmap = CreateDIBSection(hdc, &bi, DIB_RGB_COLORS, &ptrBitmapPixels, NULL, 0);
    // ^^ The output: hBitmap & ptrBitmapPixels


    // Set hBitmap in the hdcMem 
    SelectObject(hdcMem, hBitmap);



    // Set matBitmap to point to the pixels of the hBitmap
    matBitmap = Mat(y_size, x_size, CV_8UC4, ptrBitmapPixels, 0);
    //                ^^ note: first it is y, then it is x. very confusing

    // * SETUP DONE *




    // Now update the pixels using BitBlt
    BitBlt(hdcMem, 0, 0, x_size, y_size, hdcSys, 0, 0, SRCCOPY);


    // Just to do some image processing on the pixels.. (Dont have to to this)
    Mat matRef = matBitmap(Range(100, 200), Range(100, 200));
    //                              y1    y2            x1     x2
    bitwise_not(matRef, matRef); // Invert the colors in this x1,x2,y1,y2




    // Display the results through Mat
    imshow("Title", matBitmap);

    // Wait until some key is pressed

    waitKey(0);


    return 0;


}

Note that no error handling done here to make it simple to understand but you have to do error handling in your code!

Hope this helps

2
  • 1
    Why GetDC(NULL) twice?
    – Brackets
    Aug 24, 2017 at 18:15
  • Cool. It's worth noting that the code above if used in repeatable scenarios (loop for example) will leave memory leaks and will flood your heap memory / crash application in a long run. You have to call DeleteObject(hBitmap); and matBitmap.release() after you done with your image.
    – J. Wrong
    Jan 5, 2019 at 17:09

Not the answer you're looking for? Browse other questions tagged or ask your own question.