(C, C++, x86/x64 assembly): The case of forgotten return

This is a bug I once hit.

And this is also yet another demonstration, how C/C++ places return value into EAX/RAX register.

In the piece of code like that, I forgot to add "return":

#include <stdio.h>
#include <stdlib.h>

struct color
{
        int R;
        int G;
        int B;
};

struct color* create_color (int R, int G, int B)
{
        struct color* rt=(struct color*)malloc(sizeof(struct color));

        rt->R=R;
        rt->G=G;
        rt->B=B;
        // must be "return rt;" here
};

int main()
{
        struct color* a=create_color(1,2,3);
        printf ("%d %d %d\n", a->R, a->G, a->B);
};

Non-optimizing GCC 5.4 silently compiles this with no warnings. AND THE CODE WORKS! Let's see, why:

create_color:
	push	rbp
	mov	rbp, rsp
	sub	rsp, 32
	mov	DWORD PTR [rbp-20], edi
	mov	DWORD PTR [rbp-24], esi
	mov	DWORD PTR [rbp-28], edx
	mov	edi, 12
	call	malloc
; RAX is pointer to newly allocated buffer
; now fill it with R/G/B:
	mov	QWORD PTR [rbp-8], rax
	mov	rax, QWORD PTR [rbp-8]
	mov	edx, DWORD PTR [rbp-20]
	mov	DWORD PTR [rax], edx
	mov	rax, QWORD PTR [rbp-8]
	mov	edx, DWORD PTR [rbp-24]
	mov	DWORD PTR [rax+4], edx
	mov	rax, QWORD PTR [rbp-8]
	mov	edx, DWORD PTR [rbp-28]
	mov	DWORD PTR [rax+8], edx
	nop
	leave
; RAX wasn't modified till that point!
	ret


If I add "return rt;", the only instruction is added at the end, which is redundant:

create_color:
	push	rbp
	mov	rbp, rsp
	sub	rsp, 32
	mov	DWORD PTR [rbp-20], edi
	mov	DWORD PTR [rbp-24], esi
	mov	DWORD PTR [rbp-28], edx
	mov	edi, 12
	call	malloc
; RAX is pointer to buffer
	mov	QWORD PTR [rbp-8], rax
	mov	rax, QWORD PTR [rbp-8]
	mov	edx, DWORD PTR [rbp-20]
	mov	DWORD PTR [rax], edx
	mov	rax, QWORD PTR [rbp-8]
	mov	edx, DWORD PTR [rbp-24]
	mov	DWORD PTR [rax+4], edx
	mov	rax, QWORD PTR [rbp-8]
	mov	edx, DWORD PTR [rbp-28]
	mov	DWORD PTR [rax+8], edx
; reload pointer to RAX again, and this is redundant operation...
	mov	rax, QWORD PTR [rbp-8] ; new instruction
	leave
	ret


Bugs like that are very dangerous, sometimes they appear, sometimes hide. It's like: https://en.wikipedia.org/wiki/Heisenbug.

Now I'm trying optimizing GCC:

create_color:
	rep ret

main:
	xor	eax, eax
; as if create_color() was called and returned 0
	sub	rsp, 8
	mov	r8d, DWORD PTR ds:8
	mov	ecx, DWORD PTR [rax+4]
	mov	edx, DWORD PTR [rax]
	mov	esi, OFFSET FLAT:.LC1
	mov	edi, 1
	call	__printf_chk
	xor	eax, eax
	add	rsp, 8
	ret

Compiler deducing that nothing returns from the function, so it optimizes it away. And it assumes, that is returns 0 by default. The zero is then used as an address to a structure in main().. Of course, this code crashes.

GCC is C++ mode silent about it as well.

Let's try non-optimizing MSVC 2015 x86. It warns about the problem:

c:\tmp\3.c(19) : warning C4716: 'create_color': must return a value                                                               

And generates crashing code:

_rt$ = -4
_R$ = 8	
_G$ = 12
_B$ = 16
_create_color PROC
	push	ebp
	mov	ebp, esp
	push	ecx
	push	12
	call	_malloc
; EAX -> ptr to buffer
	add	esp, 4
	mov	DWORD PTR _rt$[ebp], eax
	mov	eax, DWORD PTR _rt$[ebp]
	mov	ecx, DWORD PTR _R$[ebp]
	mov	DWORD PTR [eax], ecx
	mov	edx, DWORD PTR _rt$[ebp]
	mov	eax, DWORD PTR _G$[ebp]
; EAX is set to G argument:
	mov	DWORD PTR [edx+4], eax
	mov	ecx, DWORD PTR _rt$[ebp]
	mov	edx, DWORD PTR _B$[ebp]
	mov	DWORD PTR [ecx+8], edx
	mov	esp, ebp
	pop	ebp
; EAX = G at this point:
	ret	0
_create_color ENDP


Now optimizing MSVC 2015 x86 generates crashing code as well, but for the different reason:

_a$ = -4
_main	PROC
; this is inlined optimized version of create_color():
	push	ecx
	push	12
	call	_malloc
	mov	DWORD PTR [eax], 1
	mov	DWORD PTR [eax+4], 2
	mov	DWORD PTR [eax+8], 3
; EAX -> to allocated buffer, and it's filled, OK
; now we reload ptr to buffer, thinking it's in "a" variable
; but inlined function didn't store pointer to "a" variable!
	mov	eax, DWORD PTR _a$[esp+8]
; EAX = some random garbage at this point
	push	DWORD PTR [eax+8]
	push	DWORD PTR [eax+4]
	push	DWORD PTR [eax]
	push	OFFSET $SG6074
	call	_printf
	xor	eax, eax
	add	esp, 24
	ret	0
_main	ENDP

_R$ = 8
_G$ = 12
_B$ = 16
_create_color PROC
	push	12
	call	_malloc
	mov	ecx, DWORD PTR _R$[esp]
	add	esp, 4
	mov	DWORD PTR [eax], ecx
	mov	ecx, DWORD PTR _G$[esp-4]
	mov	DWORD PTR [eax+4], ecx
	mov	ecx, DWORD PTR _B$[esp-4]
	mov	DWORD PTR [eax+8], ecx
; EAX -> to allocated buffer, OK
	ret	0
_create_color ENDP


However, non-optimizing MSVC 2015 x64 generates working code:

rt$ = 32
R$ = 64
G$ = 72
B$ = 80
create_color PROC
	mov	DWORD PTR [rsp+24], r8d
	mov	DWORD PTR [rsp+16], edx
	mov	DWORD PTR [rsp+8], ecx
	sub	rsp, 56
	mov	ecx, 12
	call	malloc
; RAX = allocated buffer
	mov	QWORD PTR rt$[rsp], rax
	mov	rax, QWORD PTR rt$[rsp]
	mov	ecx, DWORD PTR R$[rsp]
	mov	DWORD PTR [rax], ecx
	mov	rax, QWORD PTR rt$[rsp]
	mov	ecx, DWORD PTR G$[rsp]
	mov	DWORD PTR [rax+4], ecx
	mov	rax, QWORD PTR rt$[rsp]
	mov	ecx, DWORD PTR B$[rsp]
	mov	DWORD PTR [rax+8], ecx
	add	rsp, 56
; RAX didn't change down to this point
	ret	0
create_color ENDP


Optimizing MSVC 2015 x64 also inlines the function, as in case of x86, and the resulting code also crashes.

The moral of the story: warnings are very important, use "-Wall", etc, etc... When "return" statement is absent, compiler can just silently do nothing at that point.

Such a bug left unnoticed can ruin a day.

Also, shotgun debugging is bad, because again, such a bug can left unnoticed ("everything works now, so be it").


→ [list of blog posts]

Please drop me email about any bug(s) and suggestion(s): dennis(@)yurichev.com.