Do not hybrid compile and link source code by using VC and GCC!
GCC owns cross-compiling abilities in Windows. The normal environment is MinGW or Cygwin. This article focuses on the MinGW, and others are similar with it.
If comparing with the calling convention of VC, GCC has some gaps in the environment of MinGW/Cygwin. The program will be crashed if SSEx instructions are adopted in the source coded built by GCC, however they are linked with VC. It is a common case, such as many open source codes are ported from Linux.
To ease to understand these serious risks, the article will set about an example. If readers follow it step by step, the result will become obvious.
Preface:
1. foo_gcc.c/foo_gcc.h is compiled by GCC.
2. foo_vc_call_gcc.c is compiled by VC.
C Source codes:
1. foo_gcc.h/c
Header file
Source code
#ifndef _FOO_GCC_H
#define _FOO_GCC_H
#if defined( __cplusplus ) || defined( c_plusplus )
extern "C" {
#endif
#ifdef BUILD_DLL
#define GCC_API __declspec( dllexport )
#else
#define GCC_API __declspec( dllimport )
#endif
GCC_API int foo_gcc( int i );
#if defined( __cplusplus ) || defined( c_plusplus )
}
#endif
#endif
#include "foo_gcc.h"
GCC_API int foo_gcc( int i )
{
char unused[ 16 ] __attribute__ ((__aligned__ ( 16 )));
__asm__ __volatile__( "movdqa %%xmm0, %0/n/t" : "=m"(unused) );
return i * i;
}
2. foo_vc_call_gcc.c
#include "foo_gcc.h"
int foo_vc( int i )
{
__declspec( align( 16 ) ) char unused[ 16 ];
_asm
{
lea esi, unused
movdqa xmm0, xmmword ptr[ esi ]
}
i = foo_gcc( i );
return i * i;
}
int main( void )
{
foo_vc( 10 );
return 0;
}
Compile
A bash file a convenient for this boring task, and please refer it herein.
#!/bin/sh
rm ./foo_gcc.o
rm ./foo_gcc.dll
rm ./foo_gcc.s
rm ./foo_gcc.lib
rm ./libfoo_gcc.a
rm ./foo_gcc.exp
rm ./foo_gcc.def
gcc -S ./foo_gcc.c
gcc -c -O3 -DBUILD_DLL ./foo_gcc.c
gcc -shared -o ./foo_gcc.dll ./foo_gcc.o -Wl,--out-implib,./libfoo_gcc.a
pexports.exe ./foo_gcc.dll > ./foo_gcc.def
lib /machine:i386 /def:foo_gcc.def /out:foo_gcc.lib
rm ./foo_vc_call_gcc.obj
rm ./foo_vc_call_gcc.asm
rm ./foo_vc_call_gcc.exe
cl -Fa -O2 ./foo_vc_call_gcc.c ./foo_gcc.lib
Reader can refer to How to use GCC to build DLL by DEF file in MinGW? and How to generate DLL files by GCC in the MinGW? articles to build DLL by GCC in MinGW.
Execute
$ ./foo_vc_call_gcc.exe
What will be happened? I think that the program will be crashed in your system, right?!
Herein, you may be wonder why?????? Please keep patience! It is an issue of the calling convention. Before making it clear, the source codes of ASM must be investigated.
Root cause
1. foo_gcc.s
Foo_gcc.s
.file "foo_gcc.c"
.text
.globl _foo_gcc
.def _foo_gcc; .scl 2; .type 32; .endef
_foo_gcc:
pushl %ebp
movl %esp, %ebp
subl $24, %esp
/APP
# 7 "./foo_gcc.c" 1
movdqa %xmm0, -24(%ebp)
# 0 "" 2
/NO_APP
movl 8(%ebp), %eax
imull 8(%ebp), %eax
leave
ret
A. eax is used to transfer parameter. It is __stdcall calling function of DLL.
B. eax is used to return value.
C. GCC uses static strategy to align stack if SSEx instructions are used. The AMS marked by red color shows the rule(32bytes = 4bytes (returning address) + 4bytes (save ebp) + 24bytes). It hits that the esp must be aligned when invoking foo_gcc function.
2. foo_vc_call_gcc.asm
foo_icl_call_gcc.asm
; Listing generated by Microsoft (R) Optimizing Compiler Version 14.00.50727.42
TITLE c:/temp/risks/foo_vc_call_gcc.c
.686P
.XMM
include listing.inc
.model flat
INCLUDELIB LIBCMT
INCLUDELIB OLDNAMES
PUBLIC __$ArrayPad$
PUBLIC _foo_vc
EXTRN __imp__foo_gcc:PROC
EXTRN ___security_cookie:DWORD
EXTRN @__security_check_cookie@4:PROC
; Function compile flags: /Ogtpy
; COMDAT _foo_vc
_TEXT SEGMENT
_unused$ = -32 ; size = 16
__$ArrayPad$ = -4 ; size = 4
_i$ = 8 ; size = 4
_foo_vc PROC ; COMDAT
; File c:/temp/risks/foo_vc_call_gcc.c
; Line 4
push ebp
mov ebp, esp
and esp, -16 ; fffffff0H
sub esp, 44 ; 0000002cH
mov eax, DWORD PTR ___security_cookie
xor eax, esp
mov DWORD PTR __$ArrayPad$[esp+44], eax
push esi
; Line 9
lea esi, DWORD PTR _unused$[esp+48]
; Line 10
movdqa xmm0, XMMWORD PTR [esi]
; Line 12
mov eax, DWORD PTR _i$[ebp]
push eax
call DWORD PTR __imp__foo_gcc
; Line 14
imul eax, eax
; Line 15
mov ecx, DWORD PTR __$ArrayPad$[esp+52]
add esp, 4
pop esi
xor ecx, esp
call @__security_check_cookie@4
mov esp, ebp
pop ebp
ret 0
_foo_vc ENDP
_TEXT ENDS
PUBLIC __$ArrayPad$
PUBLIC _main
; Function compile flags: /Ogtpy
; COMDAT _main
_TEXT SEGMENT
_unused$699 = -32 ; size = 16
__$ArrayPad$ = -4 ; size = 4
_main PROC ; COMDAT
; Line 18
push ebp
mov ebp, esp
and esp, -16 ; fffffff0H
sub esp, 44 ; 0000002cH
mov eax, DWORD PTR ___security_cookie
xor eax, esp
mov DWORD PTR __$ArrayPad$[esp+44], eax
push esi
; Line 19
lea esi, DWORD PTR _unused$699[esp+48]
movdqa xmm0, XMMWORD PTR [esi]
push 10 ; 0000000aH
call DWORD PTR __imp__foo_gcc
; Line 22
mov ecx, DWORD PTR __$ArrayPad$[esp+52]
add esp, 4
pop esi
xor ecx, esp
xor eax, eax
call @__security_check_cookie@4
mov esp, ebp
pop ebp
ret 0
_main ENDP
_TEXT ENDS
END
A. Stack is used to transfer parameter.
B. eax is used to return value.
C. Dynamically align the address of stack. “and esp, -16” equals “and esp, 0xFFFFFFF0”, and it makes address aligned by 16bytes. However, when invoking foo_gcc function, the esp does not been aligned ((44 + 4 + 4) % 16 = 4, they are marked by red color). The operations will lead to faults of SSEx instruction, and the symptom is that program is crashed in the runtime.
Summarization:
1. The calling convention among compilers has gaps, and it will lead to crash if SSEx instructions are adopted.
2. It is a common issue, so all compilers compatible with VC have same results.
3. If the functions using SSEx instructions are not APIs of DLL, maybe the program can be run. In fact, it depends on the optimizing method of GCC. The risks are big!