Thursday 13 October 2011

What to do when sending message to some window fails or crashes the application

You want to send a message from one window to another by calling CWnd::SendMessage(...) or CWnd::PostMessage(...) and application crashes during these calls. How to find the cause of the crash?

We first need to determine the nature of the crash. Open crash dump, read message in Windows Debugger Dialog Box, read log file (if any). Functions above are called on the window through its pointer so check whether pointer is valid (not NULL - in which case you'll have access violation exception). Get window's handle and check whether it's valid (IsWindow(...)). If message is supposed to be sent to window's parent/owner, make sure that sending and receiving window are of proper type and in desired relation (parent/child or owner/owned).

It is very important to understand types of windows and relationships between them. They are all set at the time of window creation.

When creating some window with Create(...)/CreateEx(...) we need to pass two important arguments to these functions:
1) window styles
2) parent/owner window

Window styles define the nature (type) of the window and its appearance. There are three types of windows: overlapped, pop-up and child. For example, main application window is an overlapped window, which has some embedded controls (e.g. buttons) and these controls are child windows. When we press some of those buttons, a dialog box can appear and dialog box is a pop-up window. Each of these three types has its own flag within Window styles and they are defined in WinUser.hWS_OVERLAPPED (0x00000000L), WS_POPUP (0x80000000L), WS_CHILD (0x40000000L). Window style can have only one of these flags set (e.g. window cannot be pop-up and child at the same time). If you omit setting style, your window will be overlapped by default and no matter the fact you passed the pointer/handler of some window as its e.g. parent, you might not get expected behaviour as this window will not actually be of a child type!

When passing pointer or handler to parent/owner window, make sure that pointer is not NULL, and that handler is a valid handler of parent/owner at the time of the creation of child/owned window (use IsWindow()).

Do the same for window you're calling SendMessage/PostMessage on.

You can use this helper function which analyses some arbitrary window:

WndHelper.h:

#ifndef _WNDHELPER_H
#define _WNDHELPER_H

#include <afxwin.h>

// Usage:
// CWnd* pWnd = ...
// AnalyzeWnd(pWnd);
void AnalyzeWnd(const CWnd* pWnd);

#endif // _WNDHELPER_H

WndHelper.cpp:


#include "Include\WndHelper.h"

void AnalyzeWnd(const CWnd* pWnd)
{
TRACE(_T("AnalyzeWnd()"));

if(!pWnd)
{
TRACE(_T("pWnd is NULL!"));
return;
}

//
// Get Desktop
//

CWnd* pWndDesktop = pWnd->GetDesktopWindow();

if(pWndDesktop)
{
TRACE(_T("GetDesktopWindow() returned valid window pointer. pWndDesktop = %p"), pWndDesktop);

HWND hWndDesktop = pWndDesktop->GetSafeHwnd();
TRACE(_T("hWndDesktop = %p"), hWndDesktop);
}
else
{
TRACE(_T("GetDesktopWindow() returned NULL!"));
}

//
// Analyze Window
//

TRACE(_T("pWnd = %p"), pWnd);

HWND hWnd = pWnd->GetSafeHwnd();

if(::IsWindow(hWnd))
{
TRACE(_T("GetSafeHwnd() returned valid window handle. hWnd = %p"), hWnd);
}
else
{
TRACE(_T("GetSafeHwnd() returned INVALID window handle!"));
}

if(::IsWindow(pWnd->m_hWnd))
{
TRACE(_T("m_hWnd is a valid window handle. pWnd->m_hWnd = %p"), pWnd->m_hWnd);

TRACE(_T("Window styles: "));
LONG lStyle = ::GetWindowLong(pWnd->m_hWnd, GWL_STYLE);

if(lStyle == WS_OVERLAPPED) // 0
{
TRACE(_T("\t\tWS_OVERLAPPED (0)"));
}
else
{
if((lStyle & WS_POPUP) == WS_POPUP)
TRACE(_T("\t\tWS_POPUP"));

if((lStyle & WS_CHILD) == WS_CHILD)
TRACE(_T("\t\tWS_CHILD"));

if((lStyle & WS_VISIBLE) == WS_VISIBLE)
TRACE(_T("\t\tWS_VISIBLE"));

if((lStyle & WS_DISABLED) == WS_DISABLED)
TRACE(_T("\t\tWS_DISABLED"));

if((lStyle & WS_CAPTION) == WS_CAPTION)
TRACE(_T("\t\tWS_CAPTION"));

if((lStyle & WS_BORDER) == WS_BORDER)
TRACE(_T("\t\tWS_BORDER"));

if((lStyle & WS_DLGFRAME) == WS_DLGFRAME)
LOGGER_PRINTF(Debug, _T("\t\tWS_DLGFRAME"));

if((lStyle & WS_SYSMENU) == WS_SYSMENU)
TRACE(_T("\t\tWS_SYSMENU"));

// todo: add checks for other styles...
}

// todo: check extended styles
// LONG lStyleEx = GetWindowLong(GetSafeHwnd(), GWL_EXSTYLE);
}
else
{
TRACE(_T("pWnd->m_hWnd is an INVALID window handle!"));
}

hWnd = pWnd->m_hWndOwner;

if(::IsWindow(hWnd))
{
TRACE(_T("m_hWndOwner is a valid window handle. m_hWndOwner = %p"), hWnd);
}
else
{
TRACE(_T("m_hWndOwner is an INVALID window handle!"));
}

//
// Analyze Parent
//

CWnd* pWndParent = pWnd->GetParent();

if(pWndParent)
{
TRACE(_T("GetParent() returned valid pointer. pWndParent = %p"), pWndParent);

HWND hWnd = pWndParent->GetSafeHwnd();

if(::IsWindow(hWnd))
{
TRACE(_T("GetSafeHwnd() returned valid window handle. hWnd = %p"), hWnd);
}
else
{
TRACE(_T("GetSafeHwnd() returned INVALID window handle!"));
}
}
else
{
TRACE(_T("GetParent() returned NULL!"));
}

CWnd* pWndParentOwner = pWnd->GetParentOwner();

if(pWndParentOwner)
{
TRACE(_T("GetParentOwner() returned valid pointer. pWndParentOwner = %p"), pWndParentOwner);

HWND hWnd = pWndParentOwner->GetSafeHwnd();

if(::IsWindow(hWnd))
{
TRACE(_T("GetSafeHwnd() returned valid window handle. hWnd = %p"), hWnd);
}
else
{
TRACE(_T("GetSafeHwnd() returned INVALID window handle!"));
}
}
else
{
TRACE(_T("GetParentOwner() returned NULL!"));
}

//
// Analyze Owner
//

CWnd* pWndOwner = pWnd->GetOwner();

if(pWndOwner)
{
TRACE(_T("GetOwner() returned valid pointer. pWndOwner = %p"), pWndOwner);

HWND hWnd = pWndOwner->GetSafeHwnd();

if(::IsWindow(hWnd))
{
TRACE(_T("GetSafeHwnd() returned valid window handle. hWnd = %p"), hWnd);
}
else
{
TRACE(_T("GetSafeHwnd() returned INVALID window handle!"));
}
}
else
{
TRACE(_T("GetOwner() returned NULL!"));
}

CWnd* pWndSafeOwner = pWnd->GetSafeOwner();

if(pWndSafeOwner)
{
TRACE(_T("GetSafeOwner() returned valid pointer. pWndSafeOwner = %p"), pWndSafeOwner);

HWND hWnd = pWndSafeOwner->GetSafeHwnd();

if(::IsWindow(hWnd))
{
TRACE(_T("GetSafeHwnd() returned valid window handle. hWnd = %p"), hWnd);
}
else
{
TRACE(_T("GetSafeHwnd() returned INVALID window handle!"));
}
}
else
{
TRACE(_T("GetSafeOwner() returned NULL!"));
}
}

Links and references:
CWnd::SendMessage (MSDN)
CWnd::PostMessage (MSDN)

No comments: